{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "LCL_RNN_HW05_ZH.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "NYgMcc8OpP-4",
        "g6hDMXHBpP-5",
        "6DP6D9ZtpP-9",
        "A0NObpwUpP--",
        "OfVHU-YppP-_",
        "_hJ6z2ARpP_A",
        "OdElT9hhpP_A",
        "CIkLO7OqpP_B",
        "l0FdigYIpP_B",
        "fBE-S_3tpP_C",
        "ASiTDe6hpP_D",
        "vjfHpU9epP_E",
        "ue-hgaOlpP_E",
        "3vYwAMgXpP_F",
        "RTNnFqDypP_G",
        "8fd1D5mgpP_H",
        "soGFoIiypP_H",
        "OCnIeyanpP_I",
        "m7NHktqKpP_J",
        "N9uAre01pP_J",
        "TmNqHZAbpP_J",
        "iZHg8sCipP_K",
        "Dqp06bixpP_K",
        "Uy3p4-uypP_L",
        "I3QNnLRNpP_M",
        "ifHUFMbGpP_N",
        "tYkO9LltpP_O",
        "LnqAyTDDpP_P",
        "syWKeRNPpP_P",
        "5P8g6cX8pP_Q",
        "cml77b_GpP_R",
        "gs0rD5vMpP_R",
        "SiNt9FUzpP_R",
        "AGlJU7zMpP_R",
        "OVkHLwpYpP_S",
        "VIDJl34epP_T",
        "zPsxOqoJpP_T",
        "bFaBn9gypP_T",
        "k7E4Tq9NpP_U",
        "BIJ_lVn4pP_V",
        "aT-NU6mypP_V"
      ]
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bf7ce63a840349ae9bb72dced9b02515": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8520977de3cb46ef8222deb614432fac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_460407d798934975a8aab674701bd0a4",
              "IPY_MODEL_9ae42508de4c446b940d7231b6c9240a"
            ]
          }
        },
        "8520977de3cb46ef8222deb614432fac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "460407d798934975a8aab674701bd0a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_75c42e9365c84fd5b10adc531cb107e3",
            "_dom_classes": [],
            "description": "train epoch 1: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_056c2b8281154b01bc48f401d35e6dd2"
          }
        },
        "9ae42508de4c446b940d7231b6c9240a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_850f0de13e5d41cbaecb335b4952b44d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [01:54&lt;00:00,  7.44it/s, loss=6.68]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ff4ab00a6d8a4f60af13a6ac879acfd5"
          }
        },
        "75c42e9365c84fd5b10adc531cb107e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "056c2b8281154b01bc48f401d35e6dd2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "850f0de13e5d41cbaecb335b4952b44d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ff4ab00a6d8a4f60af13a6ac879acfd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d58e36f69fb849109fb31716510ae80a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_36293fde6f084386865516c1c6593e06",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_375fb18eeb964560a7e5c316043f7f53",
              "IPY_MODEL_3c55a5be326d428baba848e493f25fad"
            ]
          }
        },
        "36293fde6f084386865516c1c6593e06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "375fb18eeb964560a7e5c316043f7f53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_921e0c6ad90846b3ab36954e84de1e08",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f6fb85f005f4ac1822671190e11cc50"
          }
        },
        "3c55a5be326d428baba848e493f25fad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_229667fc0dc14a7c9e8320628203abd9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.08it/s, valid_loss=6.92]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0b5cc9f6a6f34599bc4dab5063f901f1"
          }
        },
        "921e0c6ad90846b3ab36954e84de1e08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9f6fb85f005f4ac1822671190e11cc50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "229667fc0dc14a7c9e8320628203abd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0b5cc9f6a6f34599bc4dab5063f901f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa08453f6ff34f0187434138e43c04b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2a04d4add9074aa7becf5d9dcdb3d3c9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fd9034e2397741249fd3d571925e5ff9",
              "IPY_MODEL_6b913313c5774239b84def9f6ebf165d"
            ]
          }
        },
        "2a04d4add9074aa7becf5d9dcdb3d3c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd9034e2397741249fd3d571925e5ff9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ab84246147e74efcbe1c2dbf402b9adb",
            "_dom_classes": [],
            "description": "train epoch 2: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1bd0000f9f334877bff2138630ca8385"
          }
        },
        "6b913313c5774239b84def9f6ebf165d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_850a486fa62b495cb9b2f51a533451b1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.35it/s, loss=5.84]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_25e01bad048d4b43acd0f9edfc319997"
          }
        },
        "ab84246147e74efcbe1c2dbf402b9adb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1bd0000f9f334877bff2138630ca8385": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "850a486fa62b495cb9b2f51a533451b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "25e01bad048d4b43acd0f9edfc319997": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c19f12cf4b4f41a3a0d32dfb777fab3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e2472c50cfc5438b9d273e9ab8041ab2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6ec2e45904b44e7d9d87fd9b2557ece1",
              "IPY_MODEL_2c6d05ba22cf433eb97d9c0b3d849287"
            ]
          }
        },
        "e2472c50cfc5438b9d273e9ab8041ab2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6ec2e45904b44e7d9d87fd9b2557ece1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_21e1789d219b4b3bad789d6465463971",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8e2ec4a1728741cc91e5d72e046b6bd4"
          }
        },
        "2c6d05ba22cf433eb97d9c0b3d849287": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_16aeb0e130fe40cdb85d07b475be7065",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:16&lt;00:00,  1.61it/s, valid_loss=6.5]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89c9a9d4b5dd4ca38ab9c56ac05b851c"
          }
        },
        "21e1789d219b4b3bad789d6465463971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8e2ec4a1728741cc91e5d72e046b6bd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16aeb0e130fe40cdb85d07b475be7065": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89c9a9d4b5dd4ca38ab9c56ac05b851c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bcd1ee570ae04f23a6a7d330490ceeff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9dfa528136a44470a4d6265b8400b72f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b8c09bdc4ea34d5094dfda9f27c6d69a",
              "IPY_MODEL_c2814c5274d142f08000b805904c6cb2"
            ]
          }
        },
        "9dfa528136a44470a4d6265b8400b72f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b8c09bdc4ea34d5094dfda9f27c6d69a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_af44ea3b86594c2b9645a1507abd02e3",
            "_dom_classes": [],
            "description": "train epoch 3: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4b64392ac48f40f9b99e8b4537c7ff16"
          }
        },
        "c2814c5274d142f08000b805904c6cb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_82094929eff84d92965c9204aae7a56f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [01:59&lt;00:00,  6.66it/s, loss=5.57]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3636146ec0c44ed6b1b62b32e3ebc1ef"
          }
        },
        "af44ea3b86594c2b9645a1507abd02e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4b64392ac48f40f9b99e8b4537c7ff16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82094929eff84d92965c9204aae7a56f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3636146ec0c44ed6b1b62b32e3ebc1ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2bd3f480cc7349cea11858f34ea21675": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d85e97e525ed478bb0b84ed061f2e16e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_32cad59617724dc1b4c6122bb83fad9a",
              "IPY_MODEL_d8af45522e2143d39cb800ebebbf2428"
            ]
          }
        },
        "d85e97e525ed478bb0b84ed061f2e16e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "32cad59617724dc1b4c6122bb83fad9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_845bea1ae837482cab41913de153a40d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d7735bd13b4f4b889db294fc7223ef6b"
          }
        },
        "d8af45522e2143d39cb800ebebbf2428": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c5493a4b72a845349c55c0cfbb08e6d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.14it/s, valid_loss=5.75]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5a9010d5e12e416bbab2a8ded8178a91"
          }
        },
        "845bea1ae837482cab41913de153a40d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d7735bd13b4f4b889db294fc7223ef6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5493a4b72a845349c55c0cfbb08e6d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5a9010d5e12e416bbab2a8ded8178a91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c881530aae434d2b841b8b5b234437b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a87104b984e64895b21e28341b2c3534",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b3d5d016b914408b6a7a3370922ac89",
              "IPY_MODEL_43bacc87287b49058d94638039019a63"
            ]
          }
        },
        "a87104b984e64895b21e28341b2c3534": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b3d5d016b914408b6a7a3370922ac89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_06595f5db6c84829981c417e57fcf7bb",
            "_dom_classes": [],
            "description": "train epoch 4: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_77a4df986b1c4ea380f0547192454401"
          }
        },
        "43bacc87287b49058d94638039019a63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_16f33b0870cb4e5181a3fe6675c133a0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.16it/s, loss=5.25]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_83ef198a83f74cafbc16994d1e9a1c05"
          }
        },
        "06595f5db6c84829981c417e57fcf7bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "77a4df986b1c4ea380f0547192454401": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16f33b0870cb4e5181a3fe6675c133a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "83ef198a83f74cafbc16994d1e9a1c05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae539a4968a3460c97135d3d7d301a3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_57014339c3714a58a259f0d18b6f59b5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_574997adc5ed4d2993d7c1a14b695d3b",
              "IPY_MODEL_6788d6f01a1743aea9664253a7af9b39"
            ]
          }
        },
        "57014339c3714a58a259f0d18b6f59b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "574997adc5ed4d2993d7c1a14b695d3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_85ac829588a94674ba6b0809385dea7d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4bdd1f52606f463c9fd9e1eaf64fc36b"
          }
        },
        "6788d6f01a1743aea9664253a7af9b39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d7b64268f5334ad5abdafa59015f8926",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:21&lt;00:00,  1.07s/it, valid_loss=5.55]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_62477977c74e4008b07a2ccb05e2038a"
          }
        },
        "85ac829588a94674ba6b0809385dea7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4bdd1f52606f463c9fd9e1eaf64fc36b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d7b64268f5334ad5abdafa59015f8926": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "62477977c74e4008b07a2ccb05e2038a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "41c4b30aa8b54d169d70959bca31ec53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b6f8d541e6d74b1ea7201c17012205ee",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0720fdb2ce6a4fe29b926f1db51f3496",
              "IPY_MODEL_97abf81adaeb415992c6908d7fdc3722"
            ]
          }
        },
        "b6f8d541e6d74b1ea7201c17012205ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0720fdb2ce6a4fe29b926f1db51f3496": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_431f6d9310b7496d8af06851f3847759",
            "_dom_classes": [],
            "description": "train epoch 5: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e938d9d158e94c1fb9733b994b79efff"
          }
        },
        "97abf81adaeb415992c6908d7fdc3722": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8d0a0f3875c6402ba60bac8ed34efc8f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.58it/s, loss=4.88]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_41e436b95ca04035be8e63e2d26574c2"
          }
        },
        "431f6d9310b7496d8af06851f3847759": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e938d9d158e94c1fb9733b994b79efff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d0a0f3875c6402ba60bac8ed34efc8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "41e436b95ca04035be8e63e2d26574c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6fb602fb9144735a6d3d76d9b46eda6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9386fcf485e541e5ad445b4dc8b7baa5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca2be5aef85e4586ae42f56da91ff071",
              "IPY_MODEL_6a1f16cbef39467a9464760db7267b28"
            ]
          }
        },
        "9386fcf485e541e5ad445b4dc8b7baa5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca2be5aef85e4586ae42f56da91ff071": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_beb33f900f72403b91fa3a15733112ec",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ca16aa8fc1aa4f059f9db65f6d9462ee"
          }
        },
        "6a1f16cbef39467a9464760db7267b28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c6d73576e0134ea4b357d1b1ca1e3035",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.14it/s, valid_loss=5.19]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ddb4c88579bd4ece9c14972a9dd2fad5"
          }
        },
        "beb33f900f72403b91fa3a15733112ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ca16aa8fc1aa4f059f9db65f6d9462ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c6d73576e0134ea4b357d1b1ca1e3035": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ddb4c88579bd4ece9c14972a9dd2fad5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f1b7752317241bda38a421be12e6f80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ff7fadaa45544254a2b45c7807291c65",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_76ca7ca594e64523bd4c9d1de8d9f3e9",
              "IPY_MODEL_cfb54ea1e3eb4ebcbea05f900110a14d"
            ]
          }
        },
        "ff7fadaa45544254a2b45c7807291c65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "76ca7ca594e64523bd4c9d1de8d9f3e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9554c3f20d4e45059b8e6292632ada24",
            "_dom_classes": [],
            "description": "train epoch 6: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4cb9b1b65cf4fc98b729cfdcbd2c489"
          }
        },
        "cfb54ea1e3eb4ebcbea05f900110a14d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_86366d18c58d4465b1a5d5ead41a434e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  7.39it/s, loss=4.3]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_113b92b060014e27ba14837eb3625583"
          }
        },
        "9554c3f20d4e45059b8e6292632ada24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4cb9b1b65cf4fc98b729cfdcbd2c489": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "86366d18c58d4465b1a5d5ead41a434e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "113b92b060014e27ba14837eb3625583": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b6354156b4e84d1ca00652a7fbdf2e01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_88887bce8e2448ea9c89bbd436390778",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b294449c0f254b9b83d17671ecf88df3",
              "IPY_MODEL_6f9d8687051c4a0faf9167cca6f71916"
            ]
          }
        },
        "88887bce8e2448ea9c89bbd436390778": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b294449c0f254b9b83d17671ecf88df3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_86cdcca3619e42b892c23050a0b1210d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1e6dd8e134e04b0aa1f96b40a9f590c1"
          }
        },
        "6f9d8687051c4a0faf9167cca6f71916": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f4a6b25e75c041968692358e72f733c1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:20&lt;00:00,  1.04s/it, valid_loss=4.96]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2cb100eced394578bb89e61093dbb56f"
          }
        },
        "86cdcca3619e42b892c23050a0b1210d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1e6dd8e134e04b0aa1f96b40a9f590c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f4a6b25e75c041968692358e72f733c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2cb100eced394578bb89e61093dbb56f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97a471a4f99c4dd9990a0b797be84bc0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_00073f159b7e4375addb370ec9503a41",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_bbb0260ebad648cfafaf897ac49b5238",
              "IPY_MODEL_ae1d922026e24d87b612f77a6ab79c9f"
            ]
          }
        },
        "00073f159b7e4375addb370ec9503a41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bbb0260ebad648cfafaf897ac49b5238": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d537af6594b64c03a1b60a8e1133e23d",
            "_dom_classes": [],
            "description": "train epoch 7: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c5c81d33df064bebb11c5bf0f87a25e4"
          }
        },
        "ae1d922026e24d87b612f77a6ab79c9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_82e9a03bda2d4750a9b7f1ca16f1ce44",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.28it/s, loss=4.72]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_810a2cfa6e68405b9ef4d789ffbb5f39"
          }
        },
        "d537af6594b64c03a1b60a8e1133e23d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c5c81d33df064bebb11c5bf0f87a25e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82e9a03bda2d4750a9b7f1ca16f1ce44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "810a2cfa6e68405b9ef4d789ffbb5f39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dcb20d997f1f4106a39407c0262554ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_19c2e2b53bd14728943b0a79f53ca7ba",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_188df7c8a3704727a7b9060656c52861",
              "IPY_MODEL_16c359536713446db8806380c413fe0a"
            ]
          }
        },
        "19c2e2b53bd14728943b0a79f53ca7ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "188df7c8a3704727a7b9060656c52861": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_633323e845dd41438ec3af50c30abb4f",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2b385fd6ce3c4dd390c9efeb01eeea5d"
          }
        },
        "16c359536713446db8806380c413fe0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2f562e38ee214609b33c1582adac07e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:20&lt;00:00,  1.05s/it, valid_loss=4.88]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6eb1f0cd05794ccfabb0826d949995c5"
          }
        },
        "633323e845dd41438ec3af50c30abb4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2b385fd6ce3c4dd390c9efeb01eeea5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f562e38ee214609b33c1582adac07e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6eb1f0cd05794ccfabb0826d949995c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "de2a50a480dd4706af618d60d0e7db80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f6b412def4c8469599d9b49d4dc72ac3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_937c23149ec9402f8940f3b70870920f",
              "IPY_MODEL_aab2506fe22f44bfa6debdb63d568910"
            ]
          }
        },
        "f6b412def4c8469599d9b49d4dc72ac3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "937c23149ec9402f8940f3b70870920f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5f9bfda8bd6d47aa94c55f4cfee31519",
            "_dom_classes": [],
            "description": "train epoch 8: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11a6f4d4dc6c42b390d0703f6b722b1b"
          }
        },
        "aab2506fe22f44bfa6debdb63d568910": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8d0c59e569164851a558af0a3b91afed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.08it/s, loss=4.57]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f6d617d35e54407875eadc4849aeea5"
          }
        },
        "5f9bfda8bd6d47aa94c55f4cfee31519": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11a6f4d4dc6c42b390d0703f6b722b1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d0c59e569164851a558af0a3b91afed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f6d617d35e54407875eadc4849aeea5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20078382efb14d9ea4b55fb8a40b151c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b01601a0ffb844a4acd5a18e0f8a9d64",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_10a682c72bb64c5ea0ee8ac3b7af3880",
              "IPY_MODEL_96a26fb14a3d4de395a37b2a1157d8b3"
            ]
          }
        },
        "b01601a0ffb844a4acd5a18e0f8a9d64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10a682c72bb64c5ea0ee8ac3b7af3880": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bb21650c39254c1fb5130a2a6b0911f6",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b75ec3a8369348bb8c2bc579bce8af9d"
          }
        },
        "96a26fb14a3d4de395a37b2a1157d8b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b7c9d1b307b74af19a6644e56b154e9a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.08it/s, valid_loss=4.84]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_85944c112a19478da9c6e73fe1439996"
          }
        },
        "bb21650c39254c1fb5130a2a6b0911f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b75ec3a8369348bb8c2bc579bce8af9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7c9d1b307b74af19a6644e56b154e9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "85944c112a19478da9c6e73fe1439996": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f70decbabec94008bc8be24983df879e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5316d200bc2748faadf46b900da08d82",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9e7e3499af9e4d4d9873af0bbe86de37",
              "IPY_MODEL_bedcc36b0a7640b4b30ec5d618bd85d1"
            ]
          }
        },
        "5316d200bc2748faadf46b900da08d82": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e7e3499af9e4d4d9873af0bbe86de37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_100b6643f4f34dd88a3a3caad4934f73",
            "_dom_classes": [],
            "description": "train epoch 9: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_16a47bc0ec8e4459830f2d78263b64b0"
          }
        },
        "bedcc36b0a7640b4b30ec5d618bd85d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_924e90ca1a6e4da4886314d9d8a71ad1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  5.80it/s, loss=3.85]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_716612c15c6d431c92cb86b52fd23453"
          }
        },
        "100b6643f4f34dd88a3a3caad4934f73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "16a47bc0ec8e4459830f2d78263b64b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "924e90ca1a6e4da4886314d9d8a71ad1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "716612c15c6d431c92cb86b52fd23453": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b7b7c4b74594e04b8b43c567c443859": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_67ac68a6a25a465ebd3be26ce34c50cf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9293b44a9a2c42b6bc99b99e7360a230",
              "IPY_MODEL_cab15d7b66a84cd38abf0a2ef7b837dc"
            ]
          }
        },
        "67ac68a6a25a465ebd3be26ce34c50cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9293b44a9a2c42b6bc99b99e7360a230": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3f69853a97374b35bf2e3858c915c3b8",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_23a9b478acd44fbd889c4d7999751c0f"
          }
        },
        "cab15d7b66a84cd38abf0a2ef7b837dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b749cc5317f54381b9ec8ca757fbfd0b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.10it/s, valid_loss=4.91]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9092867168354f0fbc4d7526b393e080"
          }
        },
        "3f69853a97374b35bf2e3858c915c3b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "23a9b478acd44fbd889c4d7999751c0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b749cc5317f54381b9ec8ca757fbfd0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9092867168354f0fbc4d7526b393e080": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "625f9f64009d4ec6b9aa7b872641b54c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_21a87000ff0a41df89c13f624c2c20d5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_771dcd121b6d49c1830ef432b4e3dee2",
              "IPY_MODEL_2b1ea240e5334919a27c2a4fae47559c"
            ]
          }
        },
        "21a87000ff0a41df89c13f624c2c20d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "771dcd121b6d49c1830ef432b4e3dee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a8ef656525924940a88ba6688468caa7",
            "_dom_classes": [],
            "description": "train epoch 10: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7494b9356229460db70dcdf80d82ecc0"
          }
        },
        "2b1ea240e5334919a27c2a4fae47559c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_032c223af5664afc8820f13b33725843",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:02&lt;00:00,  6.51it/s, loss=4.06]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3aaecc4a09c54fa0b008bd21f4be840d"
          }
        },
        "a8ef656525924940a88ba6688468caa7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7494b9356229460db70dcdf80d82ecc0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "032c223af5664afc8820f13b33725843": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3aaecc4a09c54fa0b008bd21f4be840d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "68574004aca64e6cbfe32784ac08658d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_15c10640d90645f293ec947af17c7879",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3a199b740a6c4ec4804dd1c6abe9b79f",
              "IPY_MODEL_dede39113f7e49c4858227ee88c88f89"
            ]
          }
        },
        "15c10640d90645f293ec947af17c7879": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3a199b740a6c4ec4804dd1c6abe9b79f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d287842f51c145158a5a8986aae88031",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_39cb72fcb6be45b297a808da85f234a0"
          }
        },
        "dede39113f7e49c4858227ee88c88f89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ed365a2e598c4de2b0fd3259ce671101",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.17it/s, valid_loss=4.77]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_55a173b9fcc24322a4b2ce13619f685a"
          }
        },
        "d287842f51c145158a5a8986aae88031": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "39cb72fcb6be45b297a808da85f234a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ed365a2e598c4de2b0fd3259ce671101": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "55a173b9fcc24322a4b2ce13619f685a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0878781c7b8493ab25445627709bf5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0fc6d82ecee948d8a46ec5d1fa2679ad",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b85a96c7c8894e17add1898ea9f4ef02",
              "IPY_MODEL_541c47357be44829a0204c5275f147d0"
            ]
          }
        },
        "0fc6d82ecee948d8a46ec5d1fa2679ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b85a96c7c8894e17add1898ea9f4ef02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_75724db3983e4194b4ef023c0a20361b",
            "_dom_classes": [],
            "description": "train epoch 11: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b1a9a4df4baa43dfa647b44d62b602ca"
          }
        },
        "541c47357be44829a0204c5275f147d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_907a81d0b8a948a2a91bfcc63a157daf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  6.87it/s, loss=4.26]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c9abe515848d441f92a2e927c67ddea4"
          }
        },
        "75724db3983e4194b4ef023c0a20361b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b1a9a4df4baa43dfa647b44d62b602ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "907a81d0b8a948a2a91bfcc63a157daf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c9abe515848d441f92a2e927c67ddea4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d432d5cd0d8a4d84a9bf9b412058fb50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2e5669550f5e45af9985dbea858ffe12",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_37d7bbcceebd4c30be5975ad925c1012",
              "IPY_MODEL_a12cbbb658c54667ac03c08c77588bb1"
            ]
          }
        },
        "2e5669550f5e45af9985dbea858ffe12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37d7bbcceebd4c30be5975ad925c1012": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ba92728f6e374c9c93c7947a7568eacb",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_df64ef963eea423c83d6a400ac3304e8"
          }
        },
        "a12cbbb658c54667ac03c08c77588bb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7010f2785e2846efb1965aff28c672eb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.12it/s, valid_loss=4.88]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dff0d17434224b5aaecad18f0785e4f2"
          }
        },
        "ba92728f6e374c9c93c7947a7568eacb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "df64ef963eea423c83d6a400ac3304e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7010f2785e2846efb1965aff28c672eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dff0d17434224b5aaecad18f0785e4f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f8a2ec038d14d7c96be1286e00884ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_325bb3604c2745469a0f74c08ca0f5a7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_82b4cbd8ca474ab08d3a3950afce4c61",
              "IPY_MODEL_678181ed5f8c40eea63d37e96413272f"
            ]
          }
        },
        "325bb3604c2745469a0f74c08ca0f5a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82b4cbd8ca474ab08d3a3950afce4c61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_99e5523babe84570ac7274328df6e55b",
            "_dom_classes": [],
            "description": "train epoch 12: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8795ae305b3478fa70cf4ba551771fc"
          }
        },
        "678181ed5f8c40eea63d37e96413272f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_194dde99d167494b95db0c5364a8849f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  6.83it/s, loss=3.95]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b5f58440390a4c148a38a10a13b66ae1"
          }
        },
        "99e5523babe84570ac7274328df6e55b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a8795ae305b3478fa70cf4ba551771fc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "194dde99d167494b95db0c5364a8849f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b5f58440390a4c148a38a10a13b66ae1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ebad828fc0341c3adb05f06d57ab6ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b6e2db9dd484e4e87a25cf5b346ac24",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_488f81747b97427b9ce7c2c2a6422783",
              "IPY_MODEL_a9e6d2bcd72f4ebd939613f0efb54ee0"
            ]
          }
        },
        "8b6e2db9dd484e4e87a25cf5b346ac24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "488f81747b97427b9ce7c2c2a6422783": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_69a16531fc4c43f2ab42767fb48db7dd",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3aa86a6045ca4068bcfa91c889aeb73a"
          }
        },
        "a9e6d2bcd72f4ebd939613f0efb54ee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d492f4d167684984bc81a37714675dd3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.24it/s, valid_loss=4.86]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ba5555e68ba742a580798662f1593544"
          }
        },
        "69a16531fc4c43f2ab42767fb48db7dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3aa86a6045ca4068bcfa91c889aeb73a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d492f4d167684984bc81a37714675dd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ba5555e68ba742a580798662f1593544": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61954000e65d45fd9045e4b1b933d624": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e961d68a700d47ab9c84f63b5ddc6198",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1b40a2d125fe45f9b1730133ca18408a",
              "IPY_MODEL_b134e83357cf4191927065f950c6fdeb"
            ]
          }
        },
        "e961d68a700d47ab9c84f63b5ddc6198": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1b40a2d125fe45f9b1730133ca18408a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e2662e636ffb41cabef0bf9f583ae13d",
            "_dom_classes": [],
            "description": "train epoch 13: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11c8906d575e4b6ca5481f43995da6af"
          }
        },
        "b134e83357cf4191927065f950c6fdeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_38cb71d4c9644946966b9696b4b332c1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  6.63it/s, loss=4.34]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_caae684f2c334ded85a0c97b4560c36c"
          }
        },
        "e2662e636ffb41cabef0bf9f583ae13d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11c8906d575e4b6ca5481f43995da6af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "38cb71d4c9644946966b9696b4b332c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "caae684f2c334ded85a0c97b4560c36c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e541d4372734057a2cd15328ad0273e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_752d8e368da4445ebd7abd220313bc5b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7e2c7d1234b048439a18bc60e0a86ccd",
              "IPY_MODEL_562241ecdf0f472098c30bf7ad417fe7"
            ]
          }
        },
        "752d8e368da4445ebd7abd220313bc5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7e2c7d1234b048439a18bc60e0a86ccd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_38b6cd024336478fb5a33d388ab689dd",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3e4d99ac3b7b4d0db3a12e1e3748f053"
          }
        },
        "562241ecdf0f472098c30bf7ad417fe7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e52589b401c244e5b9ab59c71b3fe8a6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.08it/s, valid_loss=4.92]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c16742d341a84f3f8b77759238cae471"
          }
        },
        "38b6cd024336478fb5a33d388ab689dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3e4d99ac3b7b4d0db3a12e1e3748f053": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e52589b401c244e5b9ab59c71b3fe8a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c16742d341a84f3f8b77759238cae471": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a3eb3e711b74ff5bf4a2772ee701d27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f0e53b90e6024032b4ee5314fa32395d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_129c7e6a61354f5a92c7d78f38d91a73",
              "IPY_MODEL_ee76badbc9e74e12be0db51e5339a938"
            ]
          }
        },
        "f0e53b90e6024032b4ee5314fa32395d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "129c7e6a61354f5a92c7d78f38d91a73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e752144193bb4c7ba2fd4d64cc11b625",
            "_dom_classes": [],
            "description": "train epoch 14: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d6d573f204874106988b4099726352ee"
          }
        },
        "ee76badbc9e74e12be0db51e5339a938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_793eb1209b984d39be787a3f1b29bc93",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  7.07it/s, loss=4.56]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e8c2ad59f1b54b57a97b8b0487a53f39"
          }
        },
        "e752144193bb4c7ba2fd4d64cc11b625": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d6d573f204874106988b4099726352ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "793eb1209b984d39be787a3f1b29bc93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e8c2ad59f1b54b57a97b8b0487a53f39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8cbfc946dab44b07be54ce15b78ff8f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fd62b2f46e304ccca9befdf0e61d7b0a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e18d9ad072b7433295830d6b74453627",
              "IPY_MODEL_d71a36cb16e84db4951fbf27e50a74aa"
            ]
          }
        },
        "fd62b2f46e304ccca9befdf0e61d7b0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e18d9ad072b7433295830d6b74453627": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8c66d0c5deda4aca91e21e41eeaa6b60",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c56f8250eb194ef89edd29d62a0ccb32"
          }
        },
        "d71a36cb16e84db4951fbf27e50a74aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a5c58805fd574313930201e35f62eec3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:17&lt;00:00,  1.42it/s, valid_loss=4.87]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d7c3e820b8e489d92851c6bcb8c51e0"
          }
        },
        "8c66d0c5deda4aca91e21e41eeaa6b60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c56f8250eb194ef89edd29d62a0ccb32": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a5c58805fd574313930201e35f62eec3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d7c3e820b8e489d92851c6bcb8c51e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bb8dbd0a1a524b9bb6fc442ad469579f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a192c2f7ad2f41359b4bf8e106d4c3f1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d147c646bc264669ade4f41232e9f276",
              "IPY_MODEL_cc01e4f7ecff41a184184ed6523ecba8"
            ]
          }
        },
        "a192c2f7ad2f41359b4bf8e106d4c3f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d147c646bc264669ade4f41232e9f276": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_31f995b785c9487a8593cc666ae6dc11",
            "_dom_classes": [],
            "description": "train epoch 15: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5eeaee217899423884d8b17aa1da44e1"
          }
        },
        "cc01e4f7ecff41a184184ed6523ecba8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d53b0729323044988ab3a9bac2d4e701",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:02&lt;00:00,  7.16it/s, loss=4.05]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_819de7a0797443b88a00d7b95d0ff92a"
          }
        },
        "31f995b785c9487a8593cc666ae6dc11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5eeaee217899423884d8b17aa1da44e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d53b0729323044988ab3a9bac2d4e701": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "819de7a0797443b88a00d7b95d0ff92a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "65df5040da69435ba8c20cc8c5e748c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_38fdef8bccb74d439e48c5f679e46119",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_15599c748fb040549cb730e71a962dcc",
              "IPY_MODEL_cca580d8f840464db47cf2483be5b304"
            ]
          }
        },
        "38fdef8bccb74d439e48c5f679e46119": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15599c748fb040549cb730e71a962dcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_813134bc0eb64a05909399104d58be24",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6d639facd165459b992bb9aa6860d7ef"
          }
        },
        "cca580d8f840464db47cf2483be5b304": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_29d4037ad7594c099f3803870cff0393",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.13it/s, valid_loss=4.85]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f4a47f8360074f0a9044b2af3e4d7909"
          }
        },
        "813134bc0eb64a05909399104d58be24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6d639facd165459b992bb9aa6860d7ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "29d4037ad7594c099f3803870cff0393": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f4a47f8360074f0a9044b2af3e4d7909": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4dd08c8557d047e88e47a9b167ebf612": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5ae6d2672c5049fe8c3662f8b18805ff",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4d06807549354a8bbce3e29b8914c15a",
              "IPY_MODEL_f30e26a312d9440fbeb758d530f626f1"
            ]
          }
        },
        "5ae6d2672c5049fe8c3662f8b18805ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d06807549354a8bbce3e29b8914c15a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5e8f624960b740bbb2f7b6e6c2fab9a3",
            "_dom_classes": [],
            "description": "train epoch 16: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_06de3068cbf24862a9b0fca2de4d9af4"
          }
        },
        "f30e26a312d9440fbeb758d530f626f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bd28fde6f6fb4cbe9cb189070d219d98",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:02&lt;00:00,  6.74it/s, loss=3.83]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_66ce747d8d014fad97e653eb490ce954"
          }
        },
        "5e8f624960b740bbb2f7b6e6c2fab9a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "06de3068cbf24862a9b0fca2de4d9af4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bd28fde6f6fb4cbe9cb189070d219d98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "66ce747d8d014fad97e653eb490ce954": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bec370c73f764f95a7a113ad4da26557": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_143e19329b2f469a88e24b73f530e030",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_56dfeddab5244ec6a8807a896d1f43a5",
              "IPY_MODEL_053996befd164da9b1652c7d0e3cafda"
            ]
          }
        },
        "143e19329b2f469a88e24b73f530e030": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56dfeddab5244ec6a8807a896d1f43a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_965c46e055584a82b29c616f33ebccf9",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e748f20e023041f3a0bb3b38914bf24a"
          }
        },
        "053996befd164da9b1652c7d0e3cafda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fa546018c88a4b9ea02012654895a746",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.02it/s, valid_loss=4.85]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_94f18e78e2d742b6a3c2fd27fb938259"
          }
        },
        "965c46e055584a82b29c616f33ebccf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e748f20e023041f3a0bb3b38914bf24a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa546018c88a4b9ea02012654895a746": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "94f18e78e2d742b6a3c2fd27fb938259": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52cea86d42b64e91864018672008beb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6635ea1a328f495aaa815d9a2a47cf1c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_12e309dd99664249922802bccab77da6",
              "IPY_MODEL_9598e910700d48429683ef815d595155"
            ]
          }
        },
        "6635ea1a328f495aaa815d9a2a47cf1c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12e309dd99664249922802bccab77da6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e84bb8bc3df7441ca1e7284f0c976842",
            "_dom_classes": [],
            "description": "train epoch 17: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_edd8d6181d61435b9ca85800389a64af"
          }
        },
        "9598e910700d48429683ef815d595155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f56a3b7b96e247a8b30937773c8c6d03",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.96it/s, loss=4.05]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_794ab36576cf4c2980a7ecd57ca18d2f"
          }
        },
        "e84bb8bc3df7441ca1e7284f0c976842": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "edd8d6181d61435b9ca85800389a64af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f56a3b7b96e247a8b30937773c8c6d03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "794ab36576cf4c2980a7ecd57ca18d2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf55acb845f540cf87fc0be37b864bee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_05dd8c0aee9a453f9622c4cdec1069ab",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_18cc676ba4f648819a7768d673c669f9",
              "IPY_MODEL_85240e7b8b56470b94f034a9f12f7850"
            ]
          }
        },
        "05dd8c0aee9a453f9622c4cdec1069ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "18cc676ba4f648819a7768d673c669f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1c9a18b5b564483389ad0a3cee8aaf30",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dfeb8fadffb44ab4b94eca442481ea5c"
          }
        },
        "85240e7b8b56470b94f034a9f12f7850": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_738a942c566c40638b12d8de5c4ab997",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.06it/s, valid_loss=4.85]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a62028c9b124c9da0fd3e2a5c7f80d7"
          }
        },
        "1c9a18b5b564483389ad0a3cee8aaf30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dfeb8fadffb44ab4b94eca442481ea5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "738a942c566c40638b12d8de5c4ab997": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a62028c9b124c9da0fd3e2a5c7f80d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5bc2c5b527a64a36abba837721e5f443": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b9d04ef324241a39a21bb883efdb88a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2edcb541f1ee435e9ba638ba178e7cca",
              "IPY_MODEL_3cb07cfb9d8a406e8057c62caf3f0219"
            ]
          }
        },
        "8b9d04ef324241a39a21bb883efdb88a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2edcb541f1ee435e9ba638ba178e7cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b7ed6275c7404615ba6528b2bc393ae0",
            "_dom_classes": [],
            "description": "train epoch 18: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dea52e0581f54f5f949a95be6d975cf2"
          }
        },
        "3cb07cfb9d8a406e8057c62caf3f0219": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f2cbc31443aa47adb2e74c4345bca18e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.11it/s, loss=3.73]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_292f91bd24cb44598fa47b4a3dd09119"
          }
        },
        "b7ed6275c7404615ba6528b2bc393ae0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dea52e0581f54f5f949a95be6d975cf2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f2cbc31443aa47adb2e74c4345bca18e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "292f91bd24cb44598fa47b4a3dd09119": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "915f8190942f4abaa7b4bcb02a584454": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f63a1ac3b0934f519b795f5a2f560a48",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_35f4c4d38ec84a60b6352aa99adfd89c",
              "IPY_MODEL_446ccfa1277645caa6f70e47800c9e43"
            ]
          }
        },
        "f63a1ac3b0934f519b795f5a2f560a48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "35f4c4d38ec84a60b6352aa99adfd89c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8c030566f8a44d07a682db5c8ea58955",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6509663888774b8fbf3cde536a423c3e"
          }
        },
        "446ccfa1277645caa6f70e47800c9e43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e39a78f5cace44bcb762a3b32049010d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.22it/s, valid_loss=4.8]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5fd54e07e2744882807ed3ecf857e544"
          }
        },
        "8c030566f8a44d07a682db5c8ea58955": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6509663888774b8fbf3cde536a423c3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e39a78f5cace44bcb762a3b32049010d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5fd54e07e2744882807ed3ecf857e544": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3cda324cbe904dc8b44d12a9099a5050": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2ead824d3083472c80cfb786cb4f9152",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_93134c44586c4f67b0ae7fd4ac0eba57",
              "IPY_MODEL_e8fa8437942d4281b6be0be9bcb603fc"
            ]
          }
        },
        "2ead824d3083472c80cfb786cb4f9152": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93134c44586c4f67b0ae7fd4ac0eba57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8314d74f9d4d423cb4f1432d66d9af74",
            "_dom_classes": [],
            "description": "train epoch 19: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fdf520395ee04362ada0bc8af0747446"
          }
        },
        "e8fa8437942d4281b6be0be9bcb603fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1da3f249ad7c4776932e29400b3811d2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.77it/s, loss=3.63]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_40002d05dda94fe3b8357359b91bd220"
          }
        },
        "8314d74f9d4d423cb4f1432d66d9af74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fdf520395ee04362ada0bc8af0747446": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1da3f249ad7c4776932e29400b3811d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "40002d05dda94fe3b8357359b91bd220": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20447be793d84d8cac640b5b318941f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_721f6ae801f848d3b50e8abad5e7ac36",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5d0eb472e8ea4aeb916c2df761e1b178",
              "IPY_MODEL_7dd937e8ee034b1a96605b9b421d6233"
            ]
          }
        },
        "721f6ae801f848d3b50e8abad5e7ac36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d0eb472e8ea4aeb916c2df761e1b178": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3204b85c7a274f4d8400ff4cf99abf67",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8407c09f9fb94e30bf67fabfbbb71af1"
          }
        },
        "7dd937e8ee034b1a96605b9b421d6233": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4b65a861a728450489c172bcd4f3e680",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.17it/s, valid_loss=4.83]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3696a2ec51fd44b4b06ac9cf6abf9dc8"
          }
        },
        "3204b85c7a274f4d8400ff4cf99abf67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8407c09f9fb94e30bf67fabfbbb71af1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b65a861a728450489c172bcd4f3e680": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3696a2ec51fd44b4b06ac9cf6abf9dc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9657008a39b64ba0af183af35695a0e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_848ea07702b446759d82c9c63087b70b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fdef95ddb8574fffb95809b87b776e51",
              "IPY_MODEL_527b51380a2d48cc86cbd50931575b95"
            ]
          }
        },
        "848ea07702b446759d82c9c63087b70b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fdef95ddb8574fffb95809b87b776e51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e89e732a046b4381b631da47d49c8dc3",
            "_dom_classes": [],
            "description": "train epoch 20: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cddd3eb36e7146d197965a50d42ae344"
          }
        },
        "527b51380a2d48cc86cbd50931575b95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_930dd91fdf8e4f61a7a538623cfb1762",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.13it/s, loss=3.82]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_137dd396814f4018913db9d1b41ebc3d"
          }
        },
        "e89e732a046b4381b631da47d49c8dc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cddd3eb36e7146d197965a50d42ae344": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "930dd91fdf8e4f61a7a538623cfb1762": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "137dd396814f4018913db9d1b41ebc3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c6f863a92e34b2a9c6bd51b5eaa51a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_08d57e987b9940e78cb8e5d0d285c165",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f09e513e5e41479c9a8f211945da0b18",
              "IPY_MODEL_d8835b967eab489381c661f1ababcb66"
            ]
          }
        },
        "08d57e987b9940e78cb8e5d0d285c165": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f09e513e5e41479c9a8f211945da0b18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_66acf5c9b01c41148e30640e06c90d26",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b8ced13375e742fb8ed1fe06e07adff3"
          }
        },
        "d8835b967eab489381c661f1ababcb66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5de9baf9eb12461f9594db65ed3bdbf7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.14it/s, valid_loss=4.77]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_801c33816556437b838d8c35643b1c64"
          }
        },
        "66acf5c9b01c41148e30640e06c90d26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b8ced13375e742fb8ed1fe06e07adff3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5de9baf9eb12461f9594db65ed3bdbf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "801c33816556437b838d8c35643b1c64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6aeee743f1494b29b45a92728701ed04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b9869bab632140438265580fe4c07943",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_469efd3cd8514eaba391dd69e455d611",
              "IPY_MODEL_0e5d1e5c62b84962ba2ce09ea6d1bfc8"
            ]
          }
        },
        "b9869bab632140438265580fe4c07943": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "469efd3cd8514eaba391dd69e455d611": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7cb61ffcd472416887046382c683cbe2",
            "_dom_classes": [],
            "description": "train epoch 21: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_983ca17fd9074043b5eda8d0324208f3"
          }
        },
        "0e5d1e5c62b84962ba2ce09ea6d1bfc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5369f9c47210459ba4da9c48a0ba8155",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.96it/s, loss=3.98]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f053455c34a4ddcb781a4af1eef9c0e"
          }
        },
        "7cb61ffcd472416887046382c683cbe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "983ca17fd9074043b5eda8d0324208f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5369f9c47210459ba4da9c48a0ba8155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f053455c34a4ddcb781a4af1eef9c0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b02fb5b9ae834ae0a25d7f14c945fb86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ffe811faf8e84d3bb0008d32085ce2e0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_32bf12e6fea14b9fac45e12a47fb304c",
              "IPY_MODEL_bdac37518a834f06916a77e978e0a94f"
            ]
          }
        },
        "ffe811faf8e84d3bb0008d32085ce2e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "32bf12e6fea14b9fac45e12a47fb304c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f510fad6c4944afe92c3958113a51584",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_50c6ec5c82534c428be1fdc9f7a6c236"
          }
        },
        "bdac37518a834f06916a77e978e0a94f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_52f27fd9665e45d08580061e820bc966",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.21it/s, valid_loss=4.81]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_910ed60756824e119d6881fdf14c0229"
          }
        },
        "f510fad6c4944afe92c3958113a51584": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "50c6ec5c82534c428be1fdc9f7a6c236": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52f27fd9665e45d08580061e820bc966": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "910ed60756824e119d6881fdf14c0229": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9a724a1d441142b8b9d781d59fd42fa3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_55bf2b0180084ca4903fbfd7dddf5b93",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e238771fe9554cf39fab6d48dc434506",
              "IPY_MODEL_905b2b1878f6466486c5667be993784a"
            ]
          }
        },
        "55bf2b0180084ca4903fbfd7dddf5b93": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e238771fe9554cf39fab6d48dc434506": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_40846c703cd34cb6831083dc93bd2108",
            "_dom_classes": [],
            "description": "train epoch 22: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed3fd3e8f89e42978cdee77322575d39"
          }
        },
        "905b2b1878f6466486c5667be993784a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_57508509c2a24a12beeedc88712ef957",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.79it/s, loss=4.4]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_744aa61ca0f34d0587241ab291e8a6cc"
          }
        },
        "40846c703cd34cb6831083dc93bd2108": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed3fd3e8f89e42978cdee77322575d39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57508509c2a24a12beeedc88712ef957": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "744aa61ca0f34d0587241ab291e8a6cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d190df2951f1490cbf75594585c6be63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_67c9b214a23c4477afb3b5e80ed593d0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e43f2f78a4794f7b8dd511d335c36ce6",
              "IPY_MODEL_701bcc0d6f86443ebb1ba5edd10d7e7c"
            ]
          }
        },
        "67c9b214a23c4477afb3b5e80ed593d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e43f2f78a4794f7b8dd511d335c36ce6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_995bd2b1ec884639abd1184b9f6ee023",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8672d246865460eac43ff3b504a0d75"
          }
        },
        "701bcc0d6f86443ebb1ba5edd10d7e7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_92706596069c4bd2a80331057dffcac9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.12it/s, valid_loss=4.75]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d464a38ed15041f195ab6299780255a8"
          }
        },
        "995bd2b1ec884639abd1184b9f6ee023": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a8672d246865460eac43ff3b504a0d75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "92706596069c4bd2a80331057dffcac9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d464a38ed15041f195ab6299780255a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bafdbc3df70a469c8b59158e69cbf66e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bb65d6c9bc6c486aa1dd24a364bd0421",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3e8f9222c16e489db3a4017226ce94a0",
              "IPY_MODEL_fcd0bc6ef9f94f6dabecbbff33bdacfa"
            ]
          }
        },
        "bb65d6c9bc6c486aa1dd24a364bd0421": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3e8f9222c16e489db3a4017226ce94a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_458682ce398945df963a1645a4f801d5",
            "_dom_classes": [],
            "description": "train epoch 23: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7e997fb0ae94b6294b5fd02a3f62a6f"
          }
        },
        "fcd0bc6ef9f94f6dabecbbff33bdacfa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bd6bc75cf682494a95799c0d5da48865",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.70it/s, loss=3.44]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3b6a7eedef9f4692bf506e3e3dc87eb3"
          }
        },
        "458682ce398945df963a1645a4f801d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7e997fb0ae94b6294b5fd02a3f62a6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bd6bc75cf682494a95799c0d5da48865": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3b6a7eedef9f4692bf506e3e3dc87eb3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f46a81fe38814412946d54f649cc8c90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f68ddc1853b5404e988cd3d33d1891ad",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_10536a3cd98a482d89046cbda1f01db5",
              "IPY_MODEL_b7bfa8c8aaec42a499bbf48d299ad9de"
            ]
          }
        },
        "f68ddc1853b5404e988cd3d33d1891ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10536a3cd98a482d89046cbda1f01db5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3e2cb1ad40824154b0007c7934569f55",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d68f196887ec4e3fbad1eb3b62030617"
          }
        },
        "b7bfa8c8aaec42a499bbf48d299ad9de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c8268557a7c048b998b83280c00e3764",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:17&lt;00:00,  1.27it/s, valid_loss=4.68]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e47364b656d64db09b7f32fb042c142e"
          }
        },
        "3e2cb1ad40824154b0007c7934569f55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d68f196887ec4e3fbad1eb3b62030617": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8268557a7c048b998b83280c00e3764": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e47364b656d64db09b7f32fb042c142e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7c7f6dca238412898b6b8d8ed4d9b9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4877b4bd13a943cfa3f47a99ab324fb1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_db371bd861214313b77fa769bd047f6a",
              "IPY_MODEL_10df45591374434589904fd123ec059e"
            ]
          }
        },
        "4877b4bd13a943cfa3f47a99ab324fb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db371bd861214313b77fa769bd047f6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4d8904e56e514e82aec01646bffdcebd",
            "_dom_classes": [],
            "description": "train epoch 24: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a36ebb54173245a7958028e61ca28152"
          }
        },
        "10df45591374434589904fd123ec059e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0f2186e33d5c4959b0b4d8619325502f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.59it/s, loss=3.48]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4a2962b07d140168ced4782e0e6a3ed"
          }
        },
        "4d8904e56e514e82aec01646bffdcebd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a36ebb54173245a7958028e61ca28152": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f2186e33d5c4959b0b4d8619325502f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4a2962b07d140168ced4782e0e6a3ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca1145b5312846519a7401f51052274b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c94e2f8e638b4241854e82d19a7a51db",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d54c72bc7c01408980dd2149bc9c1d48",
              "IPY_MODEL_2781a50ce0bd4e3fae8856dd5a6e928c"
            ]
          }
        },
        "c94e2f8e638b4241854e82d19a7a51db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d54c72bc7c01408980dd2149bc9c1d48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e0997af6dfc14c0f9a13bf277e1aa915",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0bc544f437994900a9f9fcc82d41df69"
          }
        },
        "2781a50ce0bd4e3fae8856dd5a6e928c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7de3ad4c157b41e7bf8f3a9681a9aef4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.16it/s, valid_loss=4.72]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_11841fc752d84106a15c754f4ef342e4"
          }
        },
        "e0997af6dfc14c0f9a13bf277e1aa915": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0bc544f437994900a9f9fcc82d41df69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7de3ad4c157b41e7bf8f3a9681a9aef4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "11841fc752d84106a15c754f4ef342e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9c1fd904008c480c87eb03a586c29bc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_217853d63f214a0fa9b50e3cd6bdc7da",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9e6369e4cee24f09a6bb942ba4fbe332",
              "IPY_MODEL_bb77626d0fee427881012af8bfac91b4"
            ]
          }
        },
        "217853d63f214a0fa9b50e3cd6bdc7da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e6369e4cee24f09a6bb942ba4fbe332": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_691b12a6249d4743afc3c410fc15e6ce",
            "_dom_classes": [],
            "description": "train epoch 25: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68a8312e677843dd8e3d457b3251839a"
          }
        },
        "bb77626d0fee427881012af8bfac91b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ce6f99a9c376445aaf9d8d8ca2b3a0a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.80it/s, loss=4.45]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_373b303fd2f84877af318c06874277a0"
          }
        },
        "691b12a6249d4743afc3c410fc15e6ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68a8312e677843dd8e3d457b3251839a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ce6f99a9c376445aaf9d8d8ca2b3a0a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "373b303fd2f84877af318c06874277a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0211a22604248319676f38dc22a78a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2d7d13000f02432f906b7328db834686",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_655e668faf4f4ca6acb15abbd84d9151",
              "IPY_MODEL_0c98b7944ea34be39a41219e368952ac"
            ]
          }
        },
        "2d7d13000f02432f906b7328db834686": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "655e668faf4f4ca6acb15abbd84d9151": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e7fa47555b8847d48b43e70c1742283d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bc33f0f7ce77479ea9ffd1fbd2f52596"
          }
        },
        "0c98b7944ea34be39a41219e368952ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_acdb71729d7e40769a1aba987c122402",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.17it/s, valid_loss=4.71]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_84569cd665b8499aa05c593d4a20fa77"
          }
        },
        "e7fa47555b8847d48b43e70c1742283d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bc33f0f7ce77479ea9ffd1fbd2f52596": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "acdb71729d7e40769a1aba987c122402": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "84569cd665b8499aa05c593d4a20fa77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6f284b48d1a485fb29f8b0081076c7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_03a4dce223334fb88da4cfe1bf9a4c5d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cfd02bdb3ff54a119169b5c316cb8671",
              "IPY_MODEL_03c57edc05f540a190b0c8d47ad91e96"
            ]
          }
        },
        "03a4dce223334fb88da4cfe1bf9a4c5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfd02bdb3ff54a119169b5c316cb8671": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c07d3d89d2d941659da4bef31762031a",
            "_dom_classes": [],
            "description": "train epoch 26: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8c5727a32ff04bab9205ca7a73d20b46"
          }
        },
        "03c57edc05f540a190b0c8d47ad91e96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_231c096ffc4248348d025d456197ce60",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.68it/s, loss=3.95]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_facb47478c44408292911eaa79488a98"
          }
        },
        "c07d3d89d2d941659da4bef31762031a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8c5727a32ff04bab9205ca7a73d20b46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "231c096ffc4248348d025d456197ce60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "facb47478c44408292911eaa79488a98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20797fca24d54f9e8fda279baa80ddf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bf621b08cd444c6e9097a6db12b79673",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_da3fc063d322459d8596107a6ef57c81",
              "IPY_MODEL_0079448623d7460898eb467523c788df"
            ]
          }
        },
        "bf621b08cd444c6e9097a6db12b79673": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da3fc063d322459d8596107a6ef57c81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cf86749023cd4bdaafb3cd107d6649bd",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2edc4f63e8a14638958d8c9120070b42"
          }
        },
        "0079448623d7460898eb467523c788df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_44940d6ccfc04b23b7b5b2fed9bfbc9a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:17&lt;00:00,  1.40it/s, valid_loss=4.68]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_74a425d701c64741bab5ed9d83fa4268"
          }
        },
        "cf86749023cd4bdaafb3cd107d6649bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2edc4f63e8a14638958d8c9120070b42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "44940d6ccfc04b23b7b5b2fed9bfbc9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "74a425d701c64741bab5ed9d83fa4268": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "18272d2a0d7f4471aff9df12797a5fae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f1c5d2eb07e341449a32bfe5872c5e0d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6e745b680dda4f58a9cfb02f75a333a5",
              "IPY_MODEL_9cfece3e5b5a4136bcb8595da192cae1"
            ]
          }
        },
        "f1c5d2eb07e341449a32bfe5872c5e0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6e745b680dda4f58a9cfb02f75a333a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e85405a881a94dd78b0b64dcae7d73cb",
            "_dom_classes": [],
            "description": "train epoch 27: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3458f2c50bb24636bd43b6cc931220ff"
          }
        },
        "9cfece3e5b5a4136bcb8595da192cae1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_388606f723ce4e83bab06dfffdee3375",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.92it/s, loss=4.48]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2ea3156d82b5489a9c6c1fd7e8e05e3c"
          }
        },
        "e85405a881a94dd78b0b64dcae7d73cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3458f2c50bb24636bd43b6cc931220ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "388606f723ce4e83bab06dfffdee3375": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2ea3156d82b5489a9c6c1fd7e8e05e3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a38bd76094564130b2d09ff8cc8ec220": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0dd333c4bf3c4bc9a3e99e6c062da52c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ebcaceef655841c2aac8836ab8e01296",
              "IPY_MODEL_21569e699f8445abaf968f25e4b8b67d"
            ]
          }
        },
        "0dd333c4bf3c4bc9a3e99e6c062da52c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ebcaceef655841c2aac8836ab8e01296": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0484ebd89b2444f1b38cfc78c1c960c0",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c66c1f7a2bca4509b54b229674bac51c"
          }
        },
        "21569e699f8445abaf968f25e4b8b67d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_09d5e48b4fba46f1ae08c47587948128",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.15it/s, valid_loss=4.68]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f43de5591174987ae60500784c9c004"
          }
        },
        "0484ebd89b2444f1b38cfc78c1c960c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c66c1f7a2bca4509b54b229674bac51c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "09d5e48b4fba46f1ae08c47587948128": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f43de5591174987ae60500784c9c004": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "762695fe4746402d8dcbbbb0bab9c42a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fe02600d4e994f8b8034d7df3ff5d45b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4c543cc65528407ca2660b61aedab21d",
              "IPY_MODEL_1807e4f5623949e0b3c10a8e8264972d"
            ]
          }
        },
        "fe02600d4e994f8b8034d7df3ff5d45b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4c543cc65528407ca2660b61aedab21d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1f210816a3fe4f1dbb6e57fc3866aff2",
            "_dom_classes": [],
            "description": "train epoch 28: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c6bb0573a5bd449ab19edcd6f5c8ad5a"
          }
        },
        "1807e4f5623949e0b3c10a8e8264972d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_28ef331c801d496082b1cb42c7d27b23",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:01&lt;00:00,  6.59it/s, loss=4.03]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ed5903b67abe49dfabb825ed004ce391"
          }
        },
        "1f210816a3fe4f1dbb6e57fc3866aff2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c6bb0573a5bd449ab19edcd6f5c8ad5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "28ef331c801d496082b1cb42c7d27b23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ed5903b67abe49dfabb825ed004ce391": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "982b55b923244a728248efecaec4e704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d990fbaec0314440b643866084ae9c5d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3d65db37c745427ab08e51232e5ab0b3",
              "IPY_MODEL_973dc1017532461d9f714c6ba539fedc"
            ]
          }
        },
        "d990fbaec0314440b643866084ae9c5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d65db37c745427ab08e51232e5ab0b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_968bee37914b4637b3afea2d39fb5d30",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dd8dfefd74ce4161a45415735d80020a"
          }
        },
        "973dc1017532461d9f714c6ba539fedc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d06ca340544b45bd803d14f5608edfc4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.13it/s, valid_loss=4.67]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_86e5b0a2022d47f689b65e6a72891729"
          }
        },
        "968bee37914b4637b3afea2d39fb5d30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dd8dfefd74ce4161a45415735d80020a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d06ca340544b45bd803d14f5608edfc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "86e5b0a2022d47f689b65e6a72891729": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d2257c0520d1499898c304ac5dac213a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2dd6beac35bb4e48ab9f718bf65834c2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5f499f87dd5a4ce99022831439ad117c",
              "IPY_MODEL_cf9bcdc00c7842d4be89055415eedb4a"
            ]
          }
        },
        "2dd6beac35bb4e48ab9f718bf65834c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f499f87dd5a4ce99022831439ad117c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_177eee6c39e0413a829a99f34f0d318b",
            "_dom_classes": [],
            "description": "train epoch 29: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_927a6da54d544e7f80c791ff23e1963e"
          }
        },
        "cf9bcdc00c7842d4be89055415eedb4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b3b6d107ff764c16ba7a10b657ec2d19",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  6.86it/s, loss=3.65]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0e9601a9899a457ca21248c5ec288918"
          }
        },
        "177eee6c39e0413a829a99f34f0d318b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "927a6da54d544e7f80c791ff23e1963e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3b6d107ff764c16ba7a10b657ec2d19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0e9601a9899a457ca21248c5ec288918": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "20ffd436d8694784a86a1e48f4e08eb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_74cb7737875e40e29439aa8907b7b2e7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b83c488b3dc4a53907c3c74f0afc3a8",
              "IPY_MODEL_1aebd94b2f414220b775910063d8a1e0"
            ]
          }
        },
        "74cb7737875e40e29439aa8907b7b2e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b83c488b3dc4a53907c3c74f0afc3a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b20cc89bc3ed44e989c1b288133e156d",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f3af9974e8140ff977cedf9ab07b3c2"
          }
        },
        "1aebd94b2f414220b775910063d8a1e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_43a026ea252f407b9fc1e06753e60dcd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:19&lt;00:00,  1.11it/s, valid_loss=4.6]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc4d5b8cd31e4bb394d4db9c68cabcb5"
          }
        },
        "b20cc89bc3ed44e989c1b288133e156d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f3af9974e8140ff977cedf9ab07b3c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "43a026ea252f407b9fc1e06753e60dcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dc4d5b8cd31e4bb394d4db9c68cabcb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "db4e2028ea3d4a1e9af1bbb3c5d87168": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9f57e4359a084f8c88844129f7bbc1b4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca119e87ad634cb39838d69d2e9c2acf",
              "IPY_MODEL_3e71dda793384021a0dd16fcf1872704"
            ]
          }
        },
        "9f57e4359a084f8c88844129f7bbc1b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca119e87ad634cb39838d69d2e9c2acf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6c49572bd81b405abf47b64af7e5bbc3",
            "_dom_classes": [],
            "description": "train epoch 30: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 791,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0e9d2488dfa48a69aeb8cfa33002479"
          }
        },
        "3e71dda793384021a0dd16fcf1872704": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_556a213f31a84dfea0d418e2317c4cac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 791/791 [02:00&lt;00:00,  7.02it/s, loss=4]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f65c0eabb4ea4afd912f5d07d59ea735"
          }
        },
        "6c49572bd81b405abf47b64af7e5bbc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0e9d2488dfa48a69aeb8cfa33002479": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "556a213f31a84dfea0d418e2317c4cac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f65c0eabb4ea4afd912f5d07d59ea735": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c5894d1718d74d66a39e4e0c4750ceb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bee6f62656df4f258886f29d5d9155f3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_afb7c009c2884685985f3bb14f12e58e",
              "IPY_MODEL_5ed9cdda8e9b453db0d4d88fbf9a0960"
            ]
          }
        },
        "bee6f62656df4f258886f29d5d9155f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "afb7c009c2884685985f3bb14f12e58e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_55a3e0378bc64aee9022510a34415912",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5e5864038e94eca81514ebd1a42a09c"
          }
        },
        "5ed9cdda8e9b453db0d4d88fbf9a0960": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ae7e2087e5104fa587a32857ca2a6b2e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.13it/s, valid_loss=4.7]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e434678700f943c58f9af8c8e7886873"
          }
        },
        "55a3e0378bc64aee9022510a34415912": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5e5864038e94eca81514ebd1a42a09c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae7e2087e5104fa587a32857ca2a6b2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e434678700f943c58f9af8c8e7886873": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1b999b1c89a4bd29c66e174a6d1dd63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b97ad6be6858426ea619cbe0ffc57de3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_791cfcee5ff64a8d8c91643e82d803ac",
              "IPY_MODEL_0a48a75124584455aeb1fc8fd64ea6fd"
            ]
          }
        },
        "b97ad6be6858426ea619cbe0ffc57de3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "791cfcee5ff64a8d8c91643e82d803ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6e3c6e0e6c8e4cc190c31ca75ba848e6",
            "_dom_classes": [],
            "description": "validation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 22,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 22,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8883c7ab13c6432890a931c54550aded"
          }
        },
        "0a48a75124584455aeb1fc8fd64ea6fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_baff250cabbd4c418cc72bd544d5c1d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 22/22 [00:18&lt;00:00,  1.18it/s, valid_loss=4.61]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22481d05b1834c29af5864445771b99c"
          }
        },
        "6e3c6e0e6c8e4cc190c31ca75ba848e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8883c7ab13c6432890a931c54550aded": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "baff250cabbd4c418cc72bd544d5c1d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22481d05b1834c29af5864445771b99c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55110f581fae40fc86a88e6a7f119b37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_97751f3c5cd840b583ee02157a63bc30",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6ac17b1b052d4d80bde3e3c285663455",
              "IPY_MODEL_2e9a65851bd34dd4b248db6608cf66d4"
            ]
          }
        },
        "97751f3c5cd840b583ee02157a63bc30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6ac17b1b052d4d80bde3e3c285663455": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e87f3d19fa4a431f9511a9b0ae6f8d72",
            "_dom_classes": [],
            "description": "prediction: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 17,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 17,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0371e2532a92493c9488b22773264212"
          }
        },
        "2e9a65851bd34dd4b248db6608cf66d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_937620760b354db1a6154f06badfe542",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 17/17 [00:15&lt;00:00,  1.08it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6ee342183b5545db8b51db81043a17d8"
          }
        },
        "e87f3d19fa4a431f9511a9b0ae6f8d72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0371e2532a92493c9488b22773264212": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "937620760b354db1a6154f06badfe542": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6ee342183b5545db8b51db81043a17d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yM44ZqEIpP-s"
      },
      "source": [
        "# **Homework 5 - Sequence-to-sequence**\n",
        "\n",
        "若有任何問題，歡迎來信至助教信箱 ntu-ml-2021spring-ta@googlegroups.com"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Yf-w71OpP-v"
      },
      "source": [
        "### (4/21 Updates)\n",
        "1. Link to reference [training curves](https://wandb.ai/george0828zhang/hw5.seq2seq.new).\n",
        "\n",
        "### (4/14 Updates) \n",
        "1. Link to tutorial [video](https://youtu.be/htG5WpZVQPU).\n",
        "2. Now defaults to load `\"avg_last_5_checkpoint.pt\"` to generate prediction.\n",
        "3. Expected run time on Colab with Tesla T4\n",
        "\n",
        "|Baseline|Details|Total Time|\n",
        "|-|:-:|:-:|\n",
        "|Simple|2m 15s $\\times$30 epochs|1hr 8m|\n",
        "|Medium|4m $\\times$30 epochs|2hr|\n",
        "|Strong|8m $\\times$30 epochs (backward)<br>+1hr (back-translation)<br>+15m $\\times$30 epochs (forward)|12hr 30m|"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iUN5huOQpP-w"
      },
      "source": [
        "# Sequence-to-Sequence 介紹\n",
        "- 大多數常見的 seq2seq model 為 encoder-decoder model，主要由兩個部分組成，分別是 encoder 和 decoder，而這兩個部可以使用 recurrent neural network (RNN)或 transformer 來實作，主要是用來解決輸入和輸出的長度不一樣的情況\n",
        "- **Encoder** 是將一連串的輸入，如文字、影片、聲音訊號等，編碼為單個向量，這單個向量可以想像為是整個輸入的抽象表示，包含了整個輸入的資訊\n",
        "- **Decoder** 是將 encoder 輸出的單個向量逐步解碼，一次輸出一個結果，直到將最後目標輸出被產生出來為止，每次輸出會影響下一次的輸出，一般會在開頭加入 \"< BOS >\" 來表示開始解碼，會在結尾輸出 \"< EOS >\" 來表示輸出結束\n",
        "\n",
        "\n",
        "![seq2seq](https://i.imgur.com/0zeDyuI.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "noP4-48ypP-x"
      },
      "source": [
        "# 作業介紹\n",
        "- 英文翻譯中文\n",
        "  - 輸入： 一句英文 （e.g.\t\ttom is a student .） \n",
        "  - 輸出： 中文翻譯 （e.g. \t\t湯姆 是 個 學生 。）\n",
        "\n",
        "- TODO\n",
        "  - 訓練一個 RNN 模型達到 Seq2seq 翻譯\n",
        "  - 訓練一個 Transformer 大幅提升效能\n",
        "  - 實作 Back-translation 大幅提升效能"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GPut4d5VpP-x"
      },
      "source": [
        "# 下載和引入需要的函式庫"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lXCjGcPYpP-y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9461b1c4-6de8-4694-da77-e9dc65d5dfa0"
      },
      "source": [
        "!pip install 'torch>=1.6.0' editdistance matplotlib sacrebleu sacremoses sentencepiece tqdm wandb\n",
        "!pip install --upgrade jupyter ipywidgets"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (1.9.0+cu102)\n",
            "Requirement already satisfied: editdistance in /usr/local/lib/python3.7/dist-packages (0.5.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-1.5.1-py3-none-any.whl (54 kB)\n",
            "\u001b[K     |████████████████████████████████| 54 kB 2.8 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 15.6 MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 42.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n",
            "Collecting wandb\n",
            "  Downloading wandb-0.11.1-py2.py3-none-any.whl (1.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7 MB 53.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6.0) (3.7.4.3)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.19.5)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.3.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from cycler>=0.10->matplotlib) (1.15.0)\n",
            "Collecting portalocker==2.0.0\n",
            "  Downloading portalocker-2.0.0-py2.py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading shortuuid-1.0.1-py3-none-any.whl (7.5 kB)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Collecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.3.1-py2.py3-none-any.whl (133 kB)\n",
            "\u001b[K     |████████████████████████████████| 133 kB 63.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.17.3)\n",
            "Collecting GitPython>=1.0.0\n",
            "  Downloading GitPython-3.1.18-py3-none-any.whl (170 kB)\n",
            "\u001b[K     |████████████████████████████████| 170 kB 63.8 MB/s \n",
            "\u001b[?25hCollecting pathtools\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "Collecting configparser>=3.8.1\n",
            "  Downloading configparser-5.0.2-py3-none-any.whl (19 kB)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Collecting graphql-core>=2.3.0\n",
            "  Downloading graphql_core-3.1.5-py3-none-any.whl (188 kB)\n",
            "\u001b[K     |████████████████████████████████| 188 kB 62.6 MB/s \n",
            "\u001b[?25hCollecting gitdb<5,>=4.0.1\n",
            "  Downloading gitdb-4.0.7-py3-none-any.whl (63 kB)\n",
            "\u001b[K     |████████████████████████████████| 63 kB 2.2 MB/s \n",
            "\u001b[?25hCollecting smmap<5,>=3.0.1\n",
            "  Downloading smmap-4.0.0-py2.py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2021.5.30)\n",
            "Building wheels for collected packages: pathtools\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8806 sha256=7734984dc546088a5a810a86e14550ee1128b8fcf12bca269633d5589b3da258\n",
            "  Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2\n",
            "Successfully built pathtools\n",
            "Installing collected packages: smmap, gitdb, shortuuid, sentry-sdk, portalocker, pathtools, graphql-core, GitPython, docker-pycreds, configparser, wandb, sentencepiece, sacremoses, sacrebleu\n",
            "Successfully installed GitPython-3.1.18 configparser-5.0.2 docker-pycreds-0.4.0 gitdb-4.0.7 graphql-core-3.1.5 pathtools-0.1.2 portalocker-2.0.0 sacrebleu-1.5.1 sacremoses-0.0.45 sentencepiece-0.1.96 sentry-sdk-1.3.1 shortuuid-1.0.1 smmap-4.0.0 wandb-0.11.1\n",
            "Requirement already satisfied: jupyter in /usr/local/lib/python3.7/dist-packages (1.0.0)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.7/dist-packages (7.6.3)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.6.1)\n",
            "Requirement already satisfied: notebook in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.3.1)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.7/dist-packages (from jupyter) (4.10.1)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.2.0)\n",
            "Requirement already satisfied: qtconsole in /usr/local/lib/python3.7/dist-packages (from jupyter) (5.1.1)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (5.1.3)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (3.5.1)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (5.5.0)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (1.0.0)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.7/dist-packages (from ipywidgets) (5.0.5)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter) (5.1.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel->jupyter) (5.3.5)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (4.4.2)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (1.0.18)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (0.8.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (57.2.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets) (2.6.1)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets) (0.2.0)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets) (2.6.0)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets) (4.7.1)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipywidgets) (1.15.0)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter) (0.10.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter) (2.11.3)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter) (1.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->jupyter) (2.8.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->jupyter) (22.1.0)\n",
            "Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook->jupyter) (0.7.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook->jupyter) (2.0.1)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (1.4.3)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.3)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.7.1)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.5.0)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (3.3.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter) (0.8.4)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter) (0.5.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter) (21.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->bleach->nbconvert->jupyter) (2.4.7)\n",
            "Requirement already satisfied: qtpy in /usr/local/lib/python3.7/dist-packages (from qtconsole->jupyter) (1.9.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NDpfV2i4pP-y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0123598c-bae2-48e9-b056-7910c4ceb152"
      },
      "source": [
        "!git clone https://github.com/pytorch/fairseq.git\n",
        "!cd fairseq && git checkout 9a1c497\n",
        "!pip install --upgrade ./fairseq/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 28901, done.\u001b[K\n",
            "remote: Counting objects: 100% (367/367), done.\u001b[K\n",
            "remote: Compressing objects: 100% (235/235), done.\u001b[K\n",
            "remote: Total 28901 (delta 168), reused 286 (delta 125), pack-reused 28534\u001b[K\n",
            "Receiving objects: 100% (28901/28901), 12.23 MiB | 23.24 MiB/s, done.\n",
            "Resolving deltas: 100% (21623/21623), done.\n",
            "Note: checking out '9a1c497'.\n",
            "\n",
            "You are in 'detached HEAD' state. You can look around, make experimental\n",
            "changes and commit them, and you can discard any commits you make in this\n",
            "state without impacting any branches by performing another checkout.\n",
            "\n",
            "If you want to create a new branch to retain commits you create, you may\n",
            "do so (now or later) by using -b with the checkout command again. Example:\n",
            "\n",
            "  git checkout -b <new-branch-name>\n",
            "\n",
            "HEAD is now at 9a1c4970 Make Hydra logging work with DDP (#1568)\n",
            "Processing ./fairseq\n",
            "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
            "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.19.5)\n",
            "Collecting hydra-core<1.1\n",
            "  Downloading hydra_core-1.0.7-py3-none-any.whl (123 kB)\n",
            "\u001b[K     |████████████████████████████████| 123 kB 7.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.5.1)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.14.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (4.41.1)\n",
            "Collecting omegaconf<2.1\n",
            "  Downloading omegaconf-2.0.6-py3-none-any.whl (36 kB)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (0.29.23)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (1.9.0+cu102)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+9a1c497) (2019.12.20)\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "  Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 11.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1->fairseq==1.0.0a0+9a1c497) (5.2.0)\n",
            "Collecting PyYAML>=5.1.*\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 13.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+9a1c497) (3.7.4.3)\n",
            "Requirement already satisfied: portalocker==2.0.0 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+9a1c497) (2.0.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+9a1c497) (2.20)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources->hydra-core<1.1->fairseq==1.0.0a0+9a1c497) (3.5.0)\n",
            "Building wheels for collected packages: fairseq, antlr4-python3-runtime\n",
            "  Building wheel for fairseq (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fairseq: filename=fairseq-1.0.0a0+9a1c497-cp37-cp37m-linux_x86_64.whl size=2851284 sha256=9bf44032c5bb919536f391845be7992f1ee0f1fd12b73e649a62b55e47f3756f\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-qp9dpao7/wheels/7c/35/80/edbd520a1a7e615df007002aeea9f6bf5f3c8f9243e072f6ce\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141229 sha256=b4c82ec24c8e613fdf3fbda0e10ef2b006c790fb63637e952d9e75878c8d0e97\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c\n",
            "Successfully built fairseq antlr4-python3-runtime\n",
            "Installing collected packages: PyYAML, omegaconf, antlr4-python3-runtime, hydra-core, fairseq\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 fairseq-1.0.0a0+9a1c497 hydra-core-1.0.7 omegaconf-2.0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_9mPzfqapP-z"
      },
      "source": [
        "import sys\n",
        "import pdb\n",
        "import pprint\n",
        "import logging\n",
        "import os\n",
        "import random\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils import data\n",
        "import numpy as np\n",
        "import tqdm.auto as tqdm\n",
        "from pathlib import Path\n",
        "from argparse import Namespace\n",
        "from fairseq import utils\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O5z6H8w-pP-z"
      },
      "source": [
        "# 設定種子"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTR4HiC4pP-0"
      },
      "source": [
        "seed = 73\n",
        "random.seed(seed)\n",
        "torch.manual_seed(seed)\n",
        "if torch.cuda.is_available():\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.cuda.manual_seed_all(seed)  \n",
        "np.random.seed(seed)  \n",
        "torch.backends.cudnn.benchmark = False\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_AzgJr-BpP-0"
      },
      "source": [
        "# 資料集介紹\n",
        "\n",
        "## 英轉繁雙語資料\n",
        "* [TED2020](#reimers-2020-multilingual-sentence-bert)\n",
        "    - 原始資料量: 398,066句    \n",
        "    - 處理後資料: 393,980句\n",
        "    \n",
        "\n",
        "## 測試資料\n",
        "- 資料量: 4,000句\n",
        "- **中文部分不公開，提供的檔案為假翻譯，全部都是句點。**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cnBKxzh7pP-1"
      },
      "source": [
        "# 資料下載"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G0hYaw_0pP-1"
      },
      "source": [
        "### 安裝megatools (optional)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "usvzjowWpP-1"
      },
      "source": [
        "#!apt-get install megatools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FbmGrn2qpP-2"
      },
      "source": [
        "## 下載檔案並解壓縮"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s3saCXi00Qwz"
      },
      "source": [
        "下载数据并解压"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R4lrO30W0P88",
        "outputId": "20956905-188f-4649-8704-5bb76ef3e6e7"
      },
      "source": [
        "!gdown --id '15f5ZghJJ1AQ9rARr4XDNCQcfxGXtzWyt' --output ted2020.tgz\n",
        "!gdown --id '18WjD1jP5p7g6E3YuAJs6PaAADvhKEbuq' --output test.tgz\n",
        "!tar -xvf ted2020.tgz \n",
        "!tar -xvf test.tgz "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=15f5ZghJJ1AQ9rARr4XDNCQcfxGXtzWyt\n",
            "To: /content/ted2020.tgz\n",
            "28.6MB [00:00, 55.6MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=18WjD1jP5p7g6E3YuAJs6PaAADvhKEbuq\n",
            "To: /content/test.tgz\n",
            "100% 150k/150k [00:00<00:00, 1.49MB/s]\n",
            "raw.en\n",
            "raw.zh\n",
            "test.en\n",
            "test.zh\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nXLHr4Li4QMi"
      },
      "source": [
        "将数据移到给定目录下"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VOgdu-Zl2re5",
        "outputId": "5c17b950-adeb-4b6d-b97e-ebc68cf65a9d"
      },
      "source": [
        "data_dir = './DATA/rawdata'\n",
        "dataset_name = 'ted2020'\n",
        "prefix = Path(data_dir).absolute() / dataset_name\n",
        "prefix.mkdir(parents=True, exist_ok=True)\n",
        "print(prefix)\n",
        "!mv {'raw.en'} {prefix/'train_dev.raw.en'}\n",
        "!mv {'raw.zh'} {prefix/'train_dev.raw.zh'}\n",
        "!mv {'test.en'} {prefix/'test.raw.en'}\n",
        "!mv {'test.zh'} {prefix/'test.raw.zh'}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/DATA/rawdata/ted2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VRYNJLjXpP-2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a91eefe-e5f9-442d-8c6f-a1d88ccb479b"
      },
      "source": [
        "'''\n",
        "data_dir = './DATA/rawdata'\n",
        "dataset_name = 'ted2020'\n",
        "urls = (\n",
        "    '\"https://onedrive.live.com/download?cid=3E549F3B24B238B4&resid=3E549F3B24B238B4%214989&authkey=AGgQ-DaR8eFSl1A\"', \n",
        "    '\"https://onedrive.live.com/download?cid=3E549F3B24B238B4&resid=3E549F3B24B238B4%214987&authkey=AA4qP_azsicwZZM\"',\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://www.csie.ntu.edu.tw/~r09922057/ML2021-hw5/ted2020.tgz\",\n",
        "#     \"https://www.csie.ntu.edu.tw/~r09922057/ML2021-hw5/test.tgz\",\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://mega.nz/#!vEcTCISJ!3Rw0eHTZWPpdHBTbQEqBDikDEdFPr7fI8WxaXK9yZ9U\",\n",
        "#     \"https://mega.nz/#!zNcnGIoJ!oPJX9AvVVs11jc0SaK6vxP_lFUNTkEcK2WbxJpvjU5Y\",\n",
        "#      \"https://drive.google.com/file/d/15f5ZghJJ1AQ9rARr4XDNCQcfxGXtzWyt/view?usp=sharing\",\n",
        "#      \"https://drive.google.com/file/d/18WjD1jP5p7g6E3YuAJs6PaAADvhKEbuq/view?usp=sharing\"\n",
        ")\n",
        "file_names = (\n",
        "    'ted2020', # train & dev\n",
        "    'test', # test\n",
        ")\n",
        "prefix = Path(data_dir).absolute() / dataset_name\n",
        "\n",
        "prefix.mkdir(parents=True, exist_ok=True)\n",
        "print(prefix)\n",
        "# for u, f in zip(urls, file_names):\n",
        "#     path = prefix/f\n",
        "#     if not path.exists():\n",
        "#         if 'mega' in u:\n",
        "#             !megadl {u} --path {path}\n",
        "#         else:\n",
        "#             !wget {u} -O {path}\n",
        "#     print(\"********************\")\n",
        "#     print(path.suffix,path,prefix)\n",
        "#     if path.suffix == \".tgz\":\n",
        "#         !tar -xvf {path} -C {prefix}\n",
        "#     elif path.suffix == \".zip\":\n",
        "#         !unzip -o {path} -d {prefix}\n",
        "!mv {prefix/'raw.en'} {prefix/'train_dev.raw.en'}\n",
        "!mv {prefix/'raw.zh'} {prefix/'train_dev.raw.zh'}\n",
        "!mv {prefix/'test.en'} {prefix/'test.raw.en'}\n",
        "!mv {prefix/'test.zh'} {prefix/'test.raw.zh'}\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/DATA/rawdata/ted2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NYgMcc8OpP-4"
      },
      "source": [
        "## 設定語言"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YH1ETbe_pP-4"
      },
      "source": [
        "src_lang = 'en'\n",
        "tgt_lang = 'zh'\n",
        "\n",
        "data_prefix = f'{prefix}/train_dev.raw'\n",
        "test_prefix = f'{prefix}/test.raw'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7CKQcy_KpP-4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "15c9cad0-fdd3-4ebb-8b6c-1e109d354bf3"
      },
      "source": [
        "!head {data_prefix+'.'+src_lang} -n 5\n",
        "!head {data_prefix+'.'+tgt_lang} -n 5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thank you so much, Chris.\n",
            "And it's truly a great honor to have the opportunity to come to this stage twice; I'm extremely grateful.\n",
            "I have been blown away by this conference, and I want to thank all of you for the many nice comments about what I had to say the other night.\n",
            "And I say that sincerely, partly because  I need that.\n",
            "Put yourselves in my position.\n",
            "非常謝謝你，克里斯。能有這個機會第二度踏上這個演講台\n",
            "真是一大榮幸。我非常感激。\n",
            "這個研討會給我留下了極為深刻的印象，我想感謝大家 對我之前演講的好評。\n",
            "我是由衷的想這麼說，有部份原因是因為 —— 我真的有需要!\n",
            "請你們設身處地為我想一想！\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g6hDMXHBpP-5"
      },
      "source": [
        "## 檔案前處理"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGSKDmDWpP-5"
      },
      "source": [
        "import re\n",
        "\n",
        "def strQ2B(ustring):\n",
        "    \"\"\"把字串全形轉半形\"\"\"\n",
        "    # 參考來源:https://ithelp.ithome.com.tw/articles/10233122\n",
        "    ss = []\n",
        "    for s in ustring:\n",
        "        rstring = \"\"\n",
        "        for uchar in s:\n",
        "            inside_code = ord(uchar)\n",
        "            if inside_code == 12288:  # 全形空格直接轉換\n",
        "                inside_code = 32\n",
        "            elif (inside_code >= 65281 and inside_code <= 65374):  # 全形字元（除空格）根據關係轉化\n",
        "                inside_code -= 65248\n",
        "            rstring += chr(inside_code)\n",
        "        ss.append(rstring)\n",
        "    return ''.join(ss)\n",
        "                \n",
        "def clean_s(s, lang):\n",
        "    if lang == 'en':\n",
        "        s = re.sub(r\"\\([^()]*\\)\", \"\", s) # remove ([text])\n",
        "        s = s.replace('-', '') # remove '-'\n",
        "        s = re.sub('([.,;!?()\\\"])', r' \\1 ', s) # keep punctuation\n",
        "    elif lang == 'zh':\n",
        "        s = strQ2B(s) # Q2B\n",
        "        s = re.sub(r\"\\([^()]*\\)\", \"\", s) # remove ([text])\n",
        "        s = s.replace(' ', '')\n",
        "        s = s.replace('—', '')\n",
        "        s = s.replace('“', '\"')\n",
        "        s = s.replace('”', '\"')\n",
        "        s = s.replace('_', '')\n",
        "        s = re.sub('([。,;!?()\\\"~「」])', r' \\1 ', s) # keep punctuation\n",
        "    s = ' '.join(s.strip().split())\n",
        "    return s\n",
        "\n",
        "def len_s(s, lang):\n",
        "    if lang == 'zh':\n",
        "        return len(s)\n",
        "    return len(s.split())\n",
        "\n",
        "def clean_corpus(prefix, l1, l2, ratio=9, max_len=1000, min_len=1):\n",
        "    if Path(f'{prefix}.clean.{l1}').exists() and Path(f'{prefix}.clean.{l2}').exists():\n",
        "        print(f'{prefix}.clean.{l1} & {l2} exists. skipping clean.')\n",
        "        return\n",
        "    with open(f'{prefix}.{l1}', 'r') as l1_in_f:\n",
        "        with open(f'{prefix}.{l2}', 'r') as l2_in_f:\n",
        "            with open(f'{prefix}.clean.{l1}', 'w') as l1_out_f:\n",
        "                with open(f'{prefix}.clean.{l2}', 'w') as l2_out_f:\n",
        "                    for s1 in l1_in_f:\n",
        "                        s1 = s1.strip()\n",
        "                        s2 = l2_in_f.readline().strip()\n",
        "                        s1 = clean_s(s1, l1)\n",
        "                        s2 = clean_s(s2, l2)\n",
        "                        s1_len = len_s(s1, l1)\n",
        "                        s2_len = len_s(s2, l2)\n",
        "                        if min_len > 0: # remove short sentence\n",
        "                            if s1_len < min_len or s2_len < min_len:\n",
        "                                continue\n",
        "                        if max_len > 0: # remove long sentence\n",
        "                            if s1_len > max_len or s2_len > max_len:\n",
        "                                continue\n",
        "                        if ratio > 0: # remove by ratio of length\n",
        "                            if s1_len/s2_len > ratio or s2_len/s1_len > ratio:\n",
        "                                continue\n",
        "                        print(s1, file=l1_out_f)\n",
        "                        print(s2, file=l2_out_f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-5Q8DfepP-7"
      },
      "source": [
        "clean_corpus(data_prefix, src_lang, tgt_lang)\n",
        "clean_corpus(test_prefix, src_lang, tgt_lang, ratio=-1, min_len=-1, max_len=-1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dkxg7njTpP-8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2e2d8edd-537a-4ac9-c3c2-b7ff48c89cda"
      },
      "source": [
        "!head {data_prefix+'.clean.'+src_lang} -n 5\n",
        "!head {data_prefix+'.clean.'+tgt_lang} -n 5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thank you so much , Chris .\n",
            "And it's truly a great honor to have the opportunity to come to this stage twice ; I'm extremely grateful .\n",
            "I have been blown away by this conference , and I want to thank all of you for the many nice comments about what I had to say the other night .\n",
            "And I say that sincerely , partly because I need that .\n",
            "Put yourselves in my position .\n",
            "非常謝謝你 , 克里斯 。 能有這個機會第二度踏上這個演講台\n",
            "真是一大榮幸 。 我非常感激 。\n",
            "這個研討會給我留下了極為深刻的印象 , 我想感謝大家對我之前演講的好評 。\n",
            "我是由衷的想這麼說 , 有部份原因是因為我真的有需要 !\n",
            "請你們設身處地為我想一想 !\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DP6D9ZtpP-9"
      },
      "source": [
        "## 切出 train/valid set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "txbQd_E7pP-9"
      },
      "source": [
        "valid_ratio = 0.01 # 3000~4000句就夠了\n",
        "train_ratio = 1 - valid_ratio"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q6GdEhJOpP-9"
      },
      "source": [
        "if (prefix/f'train.clean.{src_lang}').exists() \\\n",
        "and (prefix/f'train.clean.{tgt_lang}').exists() \\\n",
        "and (prefix/f'valid.clean.{src_lang}').exists() \\\n",
        "and (prefix/f'valid.clean.{tgt_lang}').exists():\n",
        "    print(f'train/valid splits exists. skipping split.')\n",
        "else:\n",
        "    line_num = sum(1 for line in open(f'{data_prefix}.clean.{src_lang}'))\n",
        "    labels = list(range(line_num))\n",
        "    random.shuffle(labels)\n",
        "    for lang in [src_lang, tgt_lang]:\n",
        "        train_f = open(os.path.join(data_dir, dataset_name, f'train.clean.{lang}'), 'w')\n",
        "        valid_f = open(os.path.join(data_dir, dataset_name, f'valid.clean.{lang}'), 'w')\n",
        "        count = 0\n",
        "        for line in open(f'{data_prefix}.clean.{lang}', 'r'):\n",
        "            if labels[count]/line_num < train_ratio:\n",
        "                train_f.write(line)\n",
        "            else:\n",
        "                valid_f.write(line)\n",
        "            count += 1\n",
        "        train_f.close()\n",
        "        valid_f.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0NObpwUpP--"
      },
      "source": [
        "## Subword Units \n",
        "翻譯存在的一大問題是未登錄詞(out of vocabulary)，可以使用 subword units 作為斷詞單位來解決。\n",
        "- 使用 [sentencepiece](#kudo-richardson-2018-sentencepiece) 套件\n",
        "- 用 unigram 或 byte-pair encoding (BPE)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uxSUnKzdpP--"
      },
      "source": [
        "import sentencepiece as spm\n",
        "vocab_size = 8000\n",
        "if (prefix/f'spm{vocab_size}.model').exists():\n",
        "    print(f'{prefix}/spm{vocab_size}.model exists. skipping spm_train.')\n",
        "else:\n",
        "    spm.SentencePieceTrainer.train(\n",
        "        input=','.join([f'{prefix}/train.clean.{src_lang}',\n",
        "                        f'{prefix}/valid.clean.{src_lang}',\n",
        "                        f'{prefix}/train.clean.{tgt_lang}',\n",
        "                        f'{prefix}/valid.clean.{tgt_lang}']),\n",
        "        model_prefix=prefix/f'spm{vocab_size}',\n",
        "        vocab_size=vocab_size,\n",
        "        character_coverage=1,\n",
        "        model_type='unigram', # 'bpe' 也可\n",
        "        input_sentence_size=1e6,\n",
        "        shuffle_input_sentence=True,\n",
        "        normalization_rule_name='nmt_nfkc_cf',\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BG7sTUBpP--"
      },
      "source": [
        "spm_model = spm.SentencePieceProcessor(model_file=str(prefix/f'spm{vocab_size}.model'))\n",
        "in_tag = {\n",
        "    'train': 'train.clean',\n",
        "    'valid': 'valid.clean',\n",
        "    'test': 'test.raw.clean',\n",
        "}\n",
        "for split in ['train', 'valid', 'test']:\n",
        "    for lang in [src_lang, tgt_lang]:\n",
        "        out_path = prefix/f'{split}.{lang}'\n",
        "        if out_path.exists():\n",
        "            print(f\"{out_path} exists. skipping spm_encode.\")\n",
        "        else:\n",
        "            with open(prefix/f'{split}.{lang}', 'w') as out_f:\n",
        "                with open(prefix/f'{in_tag[split]}.{lang}', 'r') as in_f:\n",
        "                    for line in in_f:\n",
        "                        line = line.strip()\n",
        "                        tok = spm_model.encode(line, out_type=str)\n",
        "                        print(' '.join(tok), file=out_f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HqI-xuo0pP-_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "936516a8-2a5a-45e8-b473-30732a248c05"
      },
      "source": [
        "!head {data_dir+'/'+dataset_name+'/train.'+src_lang} -n 5\n",
        "!head {data_dir+'/'+dataset_name+'/train.'+tgt_lang} -n 5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "▁thank ▁you ▁so ▁much ▁, ▁chris ▁.\n",
            "▁and ▁it ' s ▁ t ru ly ▁a ▁great ▁ho n or ▁to ▁have ▁the ▁ op port un ity ▁to ▁come ▁to ▁this ▁st age ▁ t wi ce ▁; ▁i ' m ▁ex t re me ly ▁gr ate ful ▁.\n",
            "▁i ▁have ▁been ▁ bl ow n ▁away ▁by ▁this ▁con fer ence ▁, ▁and ▁i ▁want ▁to ▁thank ▁all ▁of ▁you ▁for ▁the ▁many ▁ ni ce ▁ com ment s ▁about ▁what ▁i ▁had ▁to ▁say ▁the ▁other ▁night ▁.\n",
            "▁and ▁i ▁say ▁that ▁since re ly ▁, ▁part ly ▁because ▁i ▁need ▁that ▁.\n",
            "▁put ▁your s el ve s ▁in ▁my ▁po s ition ▁.\n",
            "▁ 非常 謝 謝 你 ▁, ▁ 克 里 斯 ▁。 ▁ 能 有 這個 機會 第二 度 踏 上 這個 演講 台\n",
            "▁ 真 是 一 大 榮 幸 ▁。 ▁我 非常 感 激 ▁。\n",
            "▁這個 研 討 會 給我 留 下 了 極 為 深 刻 的 印 象 ▁, ▁我想 感 謝 大家 對我 之前 演講 的 好 評 ▁。\n",
            "▁我 是由 衷 的 想 這麼 說 ▁, ▁有 部份 原因 是因為 我 真的 有 需要 ▁!\n",
            "▁ 請 你們 設 身 處 地 為 我想 一 想 ▁!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OfVHU-YppP-_"
      },
      "source": [
        "## 用 fairseq 將資料轉為 binary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0f0diELpP-_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b4791388-7b63-43fa-8210-c42533cb7715"
      },
      "source": [
        "binpath = Path('./DATA/data-bin', dataset_name)\n",
        "if binpath.exists():\n",
        "    print(binpath, \"exists, will not overwrite!\")\n",
        "else:\n",
        "    !python -m fairseq_cli.preprocess \\\n",
        "        --source-lang {src_lang}\\\n",
        "        --target-lang {tgt_lang}\\\n",
        "        --trainpref {prefix/'train'}\\\n",
        "        --validpref {prefix/'valid'}\\\n",
        "        --testpref {prefix/'test'}\\\n",
        "        --destdir {binpath}\\\n",
        "        --joined-dictionary\\\n",
        "        --workers 2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:11:12 | INFO | fairseq_cli.preprocess | Namespace(align_suffix=None, alignfile=None, all_gather_list_size=16384, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='DATA/data-bin/ted2020', empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=True, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, only_source=False, optimizer=None, padding_factor=8, profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict=None, suppress_crashes=False, target_lang='zh', task='translation', tensorboard_logdir=None, testpref='/content/DATA/rawdata/ted2020/test', tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='/content/DATA/rawdata/ted2020/train', user_dir=None, validpref='/content/DATA/rawdata/ted2020/valid', wandb_project=None, workers=2)\n",
            "2021-08-02 08:12:21 | INFO | fairseq_cli.preprocess | [en] Dictionary: 8000 types\n",
            "2021-08-02 08:13:14 | INFO | fairseq_cli.preprocess | [en] /content/DATA/rawdata/ted2020/train.en: 390041 sents, 12215608 tokens, 0.0% replaced by <unk>\n",
            "2021-08-02 08:13:14 | INFO | fairseq_cli.preprocess | [en] Dictionary: 8000 types\n",
            "2021-08-02 08:13:15 | INFO | fairseq_cli.preprocess | [en] /content/DATA/rawdata/ted2020/valid.en: 3939 sents, 122560 tokens, 0.0% replaced by <unk>\n",
            "2021-08-02 08:13:15 | INFO | fairseq_cli.preprocess | [en] Dictionary: 8000 types\n",
            "2021-08-02 08:13:16 | INFO | fairseq_cli.preprocess | [en] /content/DATA/rawdata/ted2020/test.en: 4000 sents, 122894 tokens, 0.0% replaced by <unk>\n",
            "2021-08-02 08:13:16 | INFO | fairseq_cli.preprocess | [zh] Dictionary: 8000 types\n",
            "2021-08-02 08:14:04 | INFO | fairseq_cli.preprocess | [zh] /content/DATA/rawdata/ted2020/train.zh: 390041 sents, 9533185 tokens, 0.0% replaced by <unk>\n",
            "2021-08-02 08:14:04 | INFO | fairseq_cli.preprocess | [zh] Dictionary: 8000 types\n",
            "2021-08-02 08:14:05 | INFO | fairseq_cli.preprocess | [zh] /content/DATA/rawdata/ted2020/valid.zh: 3939 sents, 96124 tokens, 0.0052% replaced by <unk>\n",
            "2021-08-02 08:14:05 | INFO | fairseq_cli.preprocess | [zh] Dictionary: 8000 types\n",
            "2021-08-02 08:14:05 | INFO | fairseq_cli.preprocess | [zh] /content/DATA/rawdata/ted2020/test.zh: 4000 sents, 8000 tokens, 0.0% replaced by <unk>\n",
            "2021-08-02 08:14:05 | INFO | fairseq_cli.preprocess | Wrote preprocessed data to DATA/data-bin/ted2020\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_hJ6z2ARpP_A"
      },
      "source": [
        "# 實驗的參數設定表"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ivt013cpP_A"
      },
      "source": [
        "config = Namespace(\n",
        "    datadir = \"./DATA/data-bin/ted2020\",\n",
        "    savedir = \"./checkpoints/rnn\",\n",
        "    source_lang = \"en\",\n",
        "    target_lang = \"zh\",\n",
        "    \n",
        "    # cpu threads when fetching & processing data.\n",
        "    num_workers=2,  \n",
        "    # batch size in terms of tokens. gradient accumulation increases the effective batchsize.\n",
        "    max_tokens=8192,\n",
        "    accum_steps=2,\n",
        "    \n",
        "    # the lr s calculated from Noam lr scheduler. you can tune the maximum lr by this factor.\n",
        "    lr_factor=2.,\n",
        "    lr_warmup=4000,\n",
        "    \n",
        "    # clipping gradient norm helps alleviate gradient exploding\n",
        "    clip_norm=1.0,\n",
        "    \n",
        "    # maximum epochs for training\n",
        "    max_epoch=30,\n",
        "    start_epoch=1,\n",
        "    \n",
        "    # beam size for beam search\n",
        "    beam=5, \n",
        "    # generate sequences of maximum length ax + b, where x is the source length\n",
        "    max_len_a=1.2, \n",
        "    max_len_b=10,\n",
        "    # when decoding, post process sentence by removing sentencepiece symbols.\n",
        "    post_process = \"sentencepiece\",\n",
        "    \n",
        "    # checkpoints\n",
        "    keep_last_epochs=5,\n",
        "    resume=None, # if resume from checkpoint name (under config.savedir)\n",
        "    \n",
        "    # logging\n",
        "    use_wandb=False,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdElT9hhpP_A"
      },
      "source": [
        "# Logging\n",
        "- logging 套件紀錄一般訊息\n",
        "- wandb 紀錄續練過程 loss, bleu, model weight 等等"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "CgMpRY4XpP_B"
      },
      "source": [
        "logging.basicConfig(\n",
        "    format=\"%(asctime)s | %(levelname)s | %(name)s | %(message)s\",\n",
        "    datefmt=\"%Y-%m-%d %H:%M:%S\",\n",
        "    level=\"INFO\", # \"DEBUG\" \"WARNING\" \"ERROR\"\n",
        "    stream=sys.stdout,\n",
        ")\n",
        "proj = \"hw5.seq2seq\"\n",
        "logger = logging.getLogger(proj)\n",
        "if config.use_wandb:\n",
        "    import wandb\n",
        "    wandb.init(project=proj, name=Path(config.savedir).stem, config=config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CIkLO7OqpP_B"
      },
      "source": [
        "# CUDA環境"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "crJIFtYYpP_B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be5386c2-f607-44e7-ffb2-bf107d56e9fc"
      },
      "source": [
        "cuda_env = utils.CudaEnvironment()\n",
        "utils.CudaEnvironment.pretty_print_cuda_env_list([cuda_env])\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:15:40 | INFO | fairseq.utils | ***********************CUDA enviroments for all 1 workers***********************\n",
            "2021-08-02 08:15:40 | INFO | fairseq.utils | rank   0: capabilities =  7.5  ; total memory = 14.756 GB ; name = Tesla T4                                \n",
            "2021-08-02 08:15:40 | INFO | fairseq.utils | ***********************CUDA enviroments for all 1 workers***********************\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l0FdigYIpP_B"
      },
      "source": [
        "# 讀取資料集"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fBE-S_3tpP_C"
      },
      "source": [
        "## 借用 fairseq 的 TranslationTask\n",
        "* 用來讀進上面 binarized 的檔案\n",
        "* 有現成的 data iterator (dataloader)\n",
        "* 字典 task.source_dictionary 和 task.target_dictionary 也很好用 \n",
        "* 有實做 beam search"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A-Do4G8GpP_C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "861010c5-cc4d-44c5-cf5b-498497959ba4"
      },
      "source": [
        "from fairseq.tasks.translation import TranslationConfig, TranslationTask\n",
        "\n",
        "## setup task\n",
        "task_cfg = TranslationConfig(\n",
        "    data=config.datadir,\n",
        "    source_lang=config.source_lang,\n",
        "    target_lang=config.target_lang,\n",
        "    train_subset=\"train\",\n",
        "    required_seq_len_multiple=8,\n",
        "    dataset_impl=\"mmap\",\n",
        "    upsample_primary=1,\n",
        ")\n",
        "task = TranslationTask.setup_task(task_cfg)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:16:06 | INFO | fairseq.tasks.translation | [en] dictionary: 8000 types\n",
            "2021-08-02 08:16:06 | INFO | fairseq.tasks.translation | [zh] dictionary: 8000 types\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MdKcECjFpP_C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "42f4d344-3016-48b5-f793-cd424e44f80f"
      },
      "source": [
        "logger.info(\"loading data for epoch 1\")\n",
        "task.load_dataset(split=\"train\", epoch=1, combine=True) # combine if you have back-translation data.\n",
        "task.load_dataset(split=\"valid\", epoch=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:16:48 | INFO | hw5.seq2seq | loading data for epoch 1\n",
            "2021-08-02 08:16:48 | INFO | fairseq.data.data_utils | loaded 390,041 examples from: ./DATA/data-bin/ted2020/train.en-zh.en\n",
            "2021-08-02 08:16:48 | INFO | fairseq.data.data_utils | loaded 390,041 examples from: ./DATA/data-bin/ted2020/train.en-zh.zh\n",
            "2021-08-02 08:16:48 | INFO | fairseq.tasks.translation | ./DATA/data-bin/ted2020 train en-zh 390041 examples\n",
            "2021-08-02 08:16:48 | INFO | fairseq.data.data_utils | loaded 3,939 examples from: ./DATA/data-bin/ted2020/valid.en-zh.en\n",
            "2021-08-02 08:16:48 | INFO | fairseq.data.data_utils | loaded 3,939 examples from: ./DATA/data-bin/ted2020/valid.en-zh.zh\n",
            "2021-08-02 08:16:48 | INFO | fairseq.tasks.translation | ./DATA/data-bin/ted2020 valid en-zh 3939 examples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vTtEqDipP_C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "103367ba-a4fb-483c-affd-287e10dfdc9a"
      },
      "source": [
        "sample = task.dataset(\"valid\")[1]\n",
        "pprint.pprint(sample)\n",
        "pprint.pprint(\n",
        "    \"Source: \" + \\\n",
        "    task.source_dictionary.string(\n",
        "        sample['source'],\n",
        "        config.post_process,\n",
        "    )\n",
        ")\n",
        "pprint.pprint(\n",
        "    \"Target: \" + \\\n",
        "    task.target_dictionary.string(\n",
        "        sample['target'],\n",
        "        config.post_process,\n",
        "    )\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'id': 1,\n",
            " 'source': tensor([  18,   14,    6, 2234,   60,   19,   80,    5,  256,   16,  405, 1407,\n",
            "        1706,    7,    2]),\n",
            " 'target': tensor([ 140,  690,   28,  270,   45,  151, 1142,  660,  606,  369, 3114, 2434,\n",
            "        1434,  192,    2])}\n",
            "\"Source: that's exactly what i do optical mind control .\"\n",
            "'Target: 這實在就是我所做的--光學操控思想'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ASiTDe6hpP_D"
      },
      "source": [
        "## Dataset Iterator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9XuOfDj4pP_D"
      },
      "source": [
        "* 將每個 batch 控制在 N 個 token 讓 GPU 記憶體更有效被利用\n",
        "* 讓 training set 每個 epoch 有不同 shuffling\n",
        "* 濾掉長度太長的句子\n",
        "* 將每個 batch 內的句子 pad 成一樣長，好讓 GPU 平行運算\n",
        "* 加上 eos 並 shift 一格\n",
        "    - teacher forcing: 為了訓練模型根據prefix生成下個字，decoder的輸入會是輸出目標序列往右shift一格。\n",
        "    - 一般是會在輸入開頭加個bos token (如下圖)\n",
        "![seq2seq](https://i.imgur.com/0zeDyuI.png)\n",
        "    - fairseq 則是直接把 eos 挪到 beginning，訓練起來效果其實差不多。例如: \n",
        "    ```\n",
        "    # 輸出目標 (target) 和 Decoder輸入 (prev_output_tokens): \n",
        "                   eos = 2\n",
        "                target = 419,  711,  238,  888,  792,   60,  968,    8,    2\n",
        "    prev_output_tokens = 2,  419,  711,  238,  888,  792,   60,  968,    8\n",
        "    ```\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R4KbGqsPpP_D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "623ea546-aeb8-4c9d-eafb-3cf7950929ce"
      },
      "source": [
        "def load_data_iterator(task, split, epoch=1, max_tokens=4000, num_workers=1, cached=True):\n",
        "    batch_iterator = task.get_batch_iterator(\n",
        "        dataset=task.dataset(split),\n",
        "        max_tokens=max_tokens,\n",
        "        max_sentences=None,\n",
        "        max_positions=utils.resolve_max_positions(\n",
        "            task.max_positions(),\n",
        "            max_tokens,\n",
        "        ),\n",
        "        ignore_invalid_inputs=True,\n",
        "        seed=seed,\n",
        "        num_workers=num_workers,\n",
        "        epoch=epoch,\n",
        "        disable_iterator_cache=not cached,\n",
        "        # Set this to False to speed up. However, if set to False, changing max_tokens beyond \n",
        "        # first call of this method has no effect. \n",
        "    )\n",
        "    return batch_iterator\n",
        "\n",
        "demo_epoch_obj = load_data_iterator(task, \"valid\", epoch=1, max_tokens=20, num_workers=1, cached=False)\n",
        "demo_iter = demo_epoch_obj.next_epoch_itr(shuffle=True)\n",
        "sample = next(demo_iter)\n",
        "sample"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:19:08 | WARNING | fairseq.tasks.fairseq_task | 2,532 samples have invalid sizes and will be skipped, max_positions=(20, 20), first few sample ids=[29, 135, 2444, 3058, 682, 731, 235, 1558, 3383, 559]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'id': tensor([723]),\n",
              " 'net_input': {'prev_output_tokens': tensor([[   2,  140,  296,  318, 1560,   51,  568,  316,  225, 1952,  254,   78,\n",
              "            151, 2691,    9,  215, 1680,   10,    1,    1,    1,    1,    1,    1]]),\n",
              "  'src_lengths': tensor([19]),\n",
              "  'src_tokens': tensor([[   1,    1,    1,    1,    1,   18,   26,   82,    8,  480,   15,  651,\n",
              "           1361,   38,    6,  176, 2696,   39,    5,  822,   92,  260,    7,    2]])},\n",
              " 'nsentences': 1,\n",
              " 'ntokens': 18,\n",
              " 'target': tensor([[ 140,  296,  318, 1560,   51,  568,  316,  225, 1952,  254,   78,  151,\n",
              "          2691,    9,  215, 1680,   10,    2,    1,    1,    1,    1,    1,    1]])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N1YRZFb1pP_D"
      },
      "source": [
        "* 每個 batch 是一個字典，key 是字串，value 是 Tensor，內容說明如下\n",
        "```python\n",
        "batch = {\n",
        "    \"id\": id, # 每個 example 的 id\n",
        "    \"nsentences\": len(samples), # batch size 句子數\n",
        "    \"ntokens\": ntokens, # batch size 字數\n",
        "    \"net_input\": {\n",
        "        \"src_tokens\": src_tokens, # 來源語言的序列\n",
        "        \"src_lengths\": src_lengths, # 每句話沒有 pad 過的長度\n",
        "        \"prev_output_tokens\": prev_output_tokens, # 上面提到右 shift 一格後的目標序列\n",
        "    },\n",
        "    \"target\": target, # 目標序列\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vjfHpU9epP_E"
      },
      "source": [
        "# 定義模型架構\n",
        "* 我們一樣繼承 fairseq 的 encoder, decoder 和 model, 這樣測試階段才能直接用他寫好的 beam search 函式"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-XGV9krpP_E"
      },
      "source": [
        "from fairseq.models import (\n",
        "    FairseqEncoder, \n",
        "    FairseqIncrementalDecoder,\n",
        "    FairseqEncoderDecoderModel\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ue-hgaOlpP_E"
      },
      "source": [
        "## Encoder 編碼器"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2w6le0K9pP_E"
      },
      "source": [
        "- seq2seq 模型的編碼器為 RNN 或 Transformer Encoder，以下說明以 RNN 為例，Transformer 略有不同。對於每個輸入，Encoder 會輸出一個向量和一個隱藏狀態(hidden state)，並將隱藏狀態用於下一個輸入。換句話說，Encoder 會逐步讀取輸入序列，並在每個 timestep 輸出單個向量，以及在最後 timestep 輸出最終隱藏狀態(content vector)\n",
        "- 參數:\n",
        "  - *args*\n",
        "      - encoder_embed_dim 是 embedding 的維度，主要將 one-hot vector 的單詞向量壓縮到指定的維度，主要是為了降維和濃縮資訊的功用\n",
        "      - encoder_ffn_embed_dim 是 RNN 輸出和隱藏狀態的維度(hidden dimension)\n",
        "      - encoder_layers 是 RNN 要疊多少層\n",
        "      - dropout 是決定有多少的機率會將某個節點變為 0，主要是為了防止 overfitting ，一般來說是在訓練時使用，測試時則不使用\n",
        "  - *dictionary*: fairseq 幫我們做好的 dictionary. 在此用來得到 padding index，好用來得到 encoder padding mask. \n",
        "  - *embed_tokens*: 事先做好的詞嵌入 (nn.Embedding)\n",
        "\n",
        "- 輸入: \n",
        "    - *src_tokens*: 英文的整數序列 e.g. 1, 28, 29, 205, 2 \n",
        "- 輸出: \n",
        "    - *outputs*: 最上層 RNN 每個 timestep 的輸出，後續可以用 Attention 再進行處理\n",
        "    - *final_hiddens*: 每層最終 timestep 的隱藏狀態，將傳遞到 Decoder 進行解碼\n",
        "    - *encoder_padding_mask*: 告訴我們哪些是位置的資訊不重要。\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rgnJIHwapP_E"
      },
      "source": [
        "class RNNEncoder(FairseqEncoder):\n",
        "    def __init__(self, args, dictionary, embed_tokens):\n",
        "        super().__init__(dictionary)\n",
        "        self.embed_tokens = embed_tokens\n",
        "        \n",
        "        self.embed_dim = args.encoder_embed_dim\n",
        "        self.hidden_dim = args.encoder_ffn_embed_dim\n",
        "        self.num_layers = args.encoder_layers\n",
        "        \n",
        "        self.dropout_in_module = nn.Dropout(args.dropout)\n",
        "        self.rnn = nn.GRU(\n",
        "            self.embed_dim, \n",
        "            self.hidden_dim, \n",
        "            self.num_layers, \n",
        "            dropout=args.dropout, \n",
        "            batch_first=False, \n",
        "            bidirectional=True\n",
        "        )\n",
        "        self.dropout_out_module = nn.Dropout(args.dropout)\n",
        "        \n",
        "        self.padding_idx = dictionary.pad()\n",
        "        \n",
        "    def combine_bidir(self, outs, bsz: int):\n",
        "        out = outs.view(self.num_layers, 2, bsz, -1).transpose(1, 2).contiguous()\n",
        "        return out.view(self.num_layers, bsz, -1)\n",
        "\n",
        "    def forward(self, src_tokens, **unused):\n",
        "        bsz, seqlen = src_tokens.size()\n",
        "        \n",
        "        # get embeddings\n",
        "        x = self.embed_tokens(src_tokens)\n",
        "        x = self.dropout_in_module(x)\n",
        "\n",
        "        # B x T x C -> T x B x C\n",
        "        x = x.transpose(0, 1)\n",
        "        \n",
        "        # 過雙向RNN\n",
        "        h0 = x.new_zeros(2 * self.num_layers, bsz, self.hidden_dim)\n",
        "        x, final_hiddens = self.rnn(x, h0)\n",
        "        outputs = self.dropout_out_module(x)\n",
        "        # outputs = [sequence len, batch size, hid dim * directions] 是最上層RNN的輸出\n",
        "        # hidden =  [num_layers * directions, batch size  , hid dim]\n",
        "        \n",
        "        # 因為 Encoder 是雙向的RNN，所以需要將同一層兩個方向的 hidden state 接在一起\n",
        "        final_hiddens = self.combine_bidir(final_hiddens, bsz)\n",
        "        # hidden =  [num_layers x batch x num_directions*hidden]\n",
        "        \n",
        "        encoder_padding_mask = src_tokens.eq(self.padding_idx).t()\n",
        "        return tuple(\n",
        "            (\n",
        "                outputs,  # seq_len x batch x hidden\n",
        "                final_hiddens,  # num_layers x batch x num_directions*hidden\n",
        "                encoder_padding_mask,  # seq_len x batch\n",
        "            )\n",
        "        )\n",
        "    \n",
        "    def reorder_encoder_out(self, encoder_out, new_order):\n",
        "        # 這個beam search時會用到，意義並不是很重要\n",
        "        return tuple(\n",
        "            (\n",
        "                encoder_out[0].index_select(1, new_order),\n",
        "                encoder_out[1].index_select(1, new_order),\n",
        "                encoder_out[2].index_select(1, new_order),\n",
        "            )\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3vYwAMgXpP_F"
      },
      "source": [
        "## Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gm5ZzawipP_F"
      },
      "source": [
        "- 當輸入過長，或是單獨靠 “content vector” 無法取得整個輸入的意思時，用 Attention Mechanism 來提供 Decoder 更多的資訊\n",
        "- 根據現在 **Decoder embeddings** ，去計算在 **Encoder outputs** 中，那些與其有較高的關係，根據關係的數值來把 Encoder outputs 平均起來作為 **Decoder** RNN 的輸入 \n",
        "- 常見 Attention 的實作是用 Neural Network / Dot Product 來算 **query** (decoder embeddings) 和 **key** (Encoder outputs) 之間的關係，再對所有算出來的數值做 **softmax** 得到分佈，最後根據這個分佈對 **values** (Encoder outputs) 做 **weight sum**\n",
        "\n",
        "- 參數:\n",
        "  - *input_embed_dim*: key 的維度，應是 decoder 要做 attend 時的向量的維度\n",
        "  - *source_embed_dim*: query 的維度，應是要被 attend 的向量(encoder outputs)的維度\n",
        "  - *output_embed_dim*: value 的維度，應是做完 attention 後，下一層預期的向量維度\n",
        "\n",
        "- 輸入: \n",
        "    - *inputs*: 就是 key，要 attend 別人的向量\n",
        "    - *encoder_outputs*: 是 query/value，被 attend 的向量\n",
        "    - *encoder_padding_mask*: 告訴我們哪些是位置的資訊不重要。\n",
        "- 輸出: \n",
        "    - *output*: 做完 attention 後的 context vector\n",
        "    - *attention score*: attention 的分布\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TTFcocCDpP_F"
      },
      "source": [
        "class AttentionLayer(nn.Module):\n",
        "    def __init__(self, input_embed_dim, source_embed_dim, output_embed_dim, bias=False):\n",
        "        super().__init__()\n",
        "\n",
        "        self.input_proj = nn.Linear(input_embed_dim, source_embed_dim, bias=bias)\n",
        "        self.output_proj = nn.Linear(\n",
        "            input_embed_dim + source_embed_dim, output_embed_dim, bias=bias\n",
        "        )\n",
        "\n",
        "    def forward(self, inputs, encoder_outputs, encoder_padding_mask):\n",
        "        # inputs: T, B, dim\n",
        "        # encoder_outputs: S x B x dim\n",
        "        # padding mask:  S x B\n",
        "        \n",
        "        # convert all to batch first\n",
        "        inputs = inputs.transpose(1,0) # B, T, dim\n",
        "        encoder_outputs = encoder_outputs.transpose(1,0) # B, S, dim\n",
        "        encoder_padding_mask = encoder_padding_mask.transpose(1,0) # B, S\n",
        "        \n",
        "        # 投影到encoder_outputs的維度\n",
        "        x = self.input_proj(inputs)\n",
        "\n",
        "        # 計算attention\n",
        "        # (B, T, dim) x (B, dim, S) = (B, T, S)\n",
        "        attn_scores = torch.bmm(x, encoder_outputs.transpose(1,2))\n",
        "\n",
        "        # 擋住padding位置的attention\n",
        "        if encoder_padding_mask is not None:\n",
        "            # 利用broadcast  B, S -> (B, 1, S)\n",
        "            encoder_padding_mask = encoder_padding_mask.unsqueeze(1)\n",
        "            attn_scores = (\n",
        "                attn_scores.float()\n",
        "                .masked_fill_(encoder_padding_mask, float(\"-inf\"))\n",
        "                .type_as(attn_scores)\n",
        "            )  # FP16 support: cast to float and back\n",
        "\n",
        "        # 在source對應維度softmax\n",
        "        attn_scores = F.softmax(attn_scores, dim=-1)\n",
        "\n",
        "        # 形狀 (B, T, S) x (B, S, dim) = (B, T, dim) 加權平均\n",
        "        x = torch.bmm(attn_scores, encoder_outputs)\n",
        "\n",
        "        # (B, T, dim)\n",
        "        x = torch.cat((x, inputs), dim=-1)\n",
        "        x = torch.tanh(self.output_proj(x)) # concat + linear + tanh\n",
        "        \n",
        "        # 回復形狀 (B, T, dim) -> (T, B, dim)\n",
        "        return x.transpose(1,0), attn_scores"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RTNnFqDypP_G"
      },
      "source": [
        "## Decoder 解碼器"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AKGfCiyXpP_G"
      },
      "source": [
        "* 解碼器的 hidden states 會用編碼器最終隱藏狀態來初始化(content vector)\n",
        "* 解碼器同時也根據目前 timestep 的輸入(也就是前幾個 timestep 的 output)，改變 hidden states，並輸出結果 \n",
        "* 如果加入 attention 可以使表現更好\n",
        "* 我們把 seq2seq 步驟寫在解碼器裡，好讓等等 Seq2Seq 這個型別可以通用 RNN 和 Transformer，而不用再改寫\n",
        "- 參數:\n",
        "  - *args*\n",
        "      - decoder_embed_dim 是解碼器 embedding 的維度，類同 encoder_embed_dim，\n",
        "      - decoder_ffn_embed_dim 是解碼器 RNN 的隱藏維度，類同 encoder_ffn_embed_dim\n",
        "      - decoder_layers 解碼器 RNN 的層數\n",
        "      - share_decoder_input_output_embed 通常 decoder 最後輸出的投影矩陣會和輸入 embedding 共用參數\n",
        "  - *dictionary*: fairseq 幫我們做好的 dictionary.\n",
        "  - *embed_tokens*: 事先做好的詞嵌入(nn.Embedding)\n",
        "- 輸入: \n",
        "    - *prev_output_tokens*: 英文的整數序列 e.g. 1, 28, 29, 205, 2 已經 shift 一格的 target\n",
        "    - *encoder_out*: 編碼器的輸出\n",
        "    - *incremental_state*: 這是測試階段為了加速，所以會記錄每個 timestep 的 hidden state 詳見 forward\n",
        "- 輸出: \n",
        "    - *outputs*: decoder 每個 timestep 的 logits，還沒經過 softmax 的分布\n",
        "    - *extra*: 沒用到"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8oLTkhv5pP_G"
      },
      "source": [
        "class RNNDecoder(FairseqIncrementalDecoder):\n",
        "    def __init__(self, args, dictionary, embed_tokens):\n",
        "        super().__init__(dictionary)\n",
        "        self.embed_tokens = embed_tokens\n",
        "        \n",
        "        assert args.decoder_layers == args.encoder_layers, f\"\"\"seq2seq rnn requires that encoder \n",
        "        and decoder have same layers of rnn. got: {args.encoder_layers, args.decoder_layers}\"\"\"\n",
        "        assert args.decoder_ffn_embed_dim == args.encoder_ffn_embed_dim*2, f\"\"\"seq2seq-rnn requires \n",
        "        that decoder hidden to be 2*encoder hidden dim. got: {args.decoder_ffn_embed_dim, args.encoder_ffn_embed_dim*2}\"\"\"\n",
        "        \n",
        "        self.embed_dim = args.decoder_embed_dim\n",
        "        self.hidden_dim = args.decoder_ffn_embed_dim\n",
        "        self.num_layers = args.decoder_layers\n",
        "        \n",
        "        \n",
        "        self.dropout_in_module = nn.Dropout(args.dropout)\n",
        "        self.rnn = nn.GRU(\n",
        "            self.embed_dim, \n",
        "            self.hidden_dim, \n",
        "            self.num_layers, \n",
        "            dropout=args.dropout, \n",
        "            batch_first=False, \n",
        "            bidirectional=False\n",
        "        )\n",
        "        self.attention = AttentionLayer(\n",
        "            self.embed_dim, self.hidden_dim, self.embed_dim, bias=False\n",
        "        ) \n",
        "        # self.attention = None\n",
        "        self.dropout_out_module = nn.Dropout(args.dropout)\n",
        "        \n",
        "        if self.hidden_dim != self.embed_dim:\n",
        "            self.project_out_dim = nn.Linear(self.hidden_dim, self.embed_dim)\n",
        "        else:\n",
        "            self.project_out_dim = None\n",
        "        \n",
        "        if args.share_decoder_input_output_embed:\n",
        "            self.output_projection = nn.Linear(\n",
        "                self.embed_tokens.weight.shape[1],\n",
        "                self.embed_tokens.weight.shape[0],\n",
        "                bias=False,\n",
        "            )\n",
        "            self.output_projection.weight = self.embed_tokens.weight\n",
        "        else:\n",
        "            self.output_projection = nn.Linear(\n",
        "                self.output_embed_dim, len(dictionary), bias=False\n",
        "            )\n",
        "            nn.init.normal_(\n",
        "                self.output_projection.weight, mean=0, std=self.output_embed_dim ** -0.5\n",
        "            )\n",
        "        \n",
        "    def forward(self, prev_output_tokens, encoder_out, incremental_state=None, **unused):\n",
        "        # 取出encoder的輸出\n",
        "        encoder_outputs, encoder_hiddens, encoder_padding_mask = encoder_out\n",
        "        # outputs:          seq_len x batch x num_directions*hidden\n",
        "        # encoder_hiddens:  num_layers x batch x num_directions*encoder_hidden\n",
        "        # padding_mask:     seq_len x batch\n",
        "        \n",
        "        if incremental_state is not None and len(incremental_state) > 0:\n",
        "            # 有上個timestep留下的資訊，讀進來就可以繼續decode，不用從bos重來\n",
        "            prev_output_tokens = prev_output_tokens[:, -1:]\n",
        "            cache_state = self.get_incremental_state(incremental_state, \"cached_state\")\n",
        "            prev_hiddens = cache_state[\"prev_hiddens\"]\n",
        "        else:\n",
        "            # 沒有incremental state代表這是training或者是test time時的第一步\n",
        "            # 準備seq2seq: 把encoder_hiddens pass進去decoder的hidden states\n",
        "            prev_hiddens = encoder_hiddens\n",
        "        \n",
        "        bsz, seqlen = prev_output_tokens.size()\n",
        "        \n",
        "        # embed tokens\n",
        "        x = self.embed_tokens(prev_output_tokens)\n",
        "        x = self.dropout_in_module(x)\n",
        "\n",
        "        # B x T x C -> T x B x C\n",
        "        x = x.transpose(0, 1)\n",
        "                \n",
        "        # 做decoder-to-encoder attention\n",
        "        if self.attention is not None:\n",
        "            x, attn = self.attention(x, encoder_outputs, encoder_padding_mask)\n",
        "                        \n",
        "        # 過單向RNN\n",
        "        x, final_hiddens = self.rnn(x, prev_hiddens)\n",
        "        # outputs = [sequence len, batch size, hid dim]\n",
        "        # hidden =  [num_layers * directions, batch size  , hid dim]\n",
        "        x = self.dropout_out_module(x)\n",
        "                \n",
        "        # 投影到embedding size (如果hidden 和embed size不一樣，然後share_embedding又設成True,需要額外project一次)\n",
        "        if self.project_out_dim != None:\n",
        "            x = self.project_out_dim(x)\n",
        "        \n",
        "        # 投影到vocab size 的分佈\n",
        "        x = self.output_projection(x)\n",
        "        \n",
        "        # T x B x C -> B x T x C\n",
        "        x = x.transpose(1, 0)\n",
        "        \n",
        "        # 如果是Incremental, 記錄這個timestep的hidden states, 下個timestep讀回來\n",
        "        cache_state = {\n",
        "            \"prev_hiddens\": final_hiddens,\n",
        "        }\n",
        "        self.set_incremental_state(incremental_state, \"cached_state\", cache_state)\n",
        "        \n",
        "        return x, None\n",
        "    \n",
        "    def reorder_incremental_state(\n",
        "        self,\n",
        "        incremental_state,\n",
        "        new_order,\n",
        "    ):\n",
        "        # 這個beam search時會用到，意義並不是很重要\n",
        "        cache_state = self.get_incremental_state(incremental_state, \"cached_state\")\n",
        "        prev_hiddens = cache_state[\"prev_hiddens\"]\n",
        "        prev_hiddens = [p.index_select(0, new_order) for p in prev_hiddens]\n",
        "        cache_state = {\n",
        "            \"prev_hiddens\": torch.stack(prev_hiddens),\n",
        "        }\n",
        "        self.set_incremental_state(incremental_state, \"cached_state\", cache_state)\n",
        "        return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8fd1D5mgpP_H"
      },
      "source": [
        "## Seq2Seq\n",
        "- 由 **Encoder** 和 **Decoder** 組成\n",
        "- 接收輸入並傳給 **Encoder** \n",
        "- 將 **Encoder** 的輸出傳給 **Decoder**\n",
        "- **Decoder** 根據前幾個 timestep 的輸出和 **Encoder** 輸出進行解碼  \n",
        "- 當解碼完成後，將 **Decoder** 的輸出傳回 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HmEDx7D-pP_H"
      },
      "source": [
        "class Seq2Seq(FairseqEncoderDecoderModel):\n",
        "    def __init__(self, args, encoder, decoder):\n",
        "        super().__init__(encoder, decoder)\n",
        "        self.args = args\n",
        "    \n",
        "    def forward(\n",
        "        self,\n",
        "        src_tokens,\n",
        "        src_lengths,\n",
        "        prev_output_tokens,\n",
        "        return_all_hiddens: bool = True,\n",
        "    ):\n",
        "        \"\"\"\n",
        "        Run the forward pass for an encoder-decoder model.\n",
        "        \"\"\"\n",
        "        encoder_out = self.encoder(\n",
        "            src_tokens, src_lengths=src_lengths, return_all_hiddens=return_all_hiddens\n",
        "        )\n",
        "        logits, extra = self.decoder(\n",
        "            prev_output_tokens,\n",
        "            encoder_out=encoder_out,\n",
        "            src_lengths=src_lengths,\n",
        "            return_all_hiddens=return_all_hiddens,\n",
        "        )\n",
        "        return logits, extra"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "soGFoIiypP_H"
      },
      "source": [
        "# 模型初始化"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1SSMrK92pP_H"
      },
      "source": [
        "# # HINT: transformer 架構\n",
        "# from fairseq.models.transformer import (\n",
        "#     TransformerEncoder, \n",
        "#     TransformerDecoder,\n",
        "# )\n",
        "\n",
        "def build_model(args, task):\n",
        "    \"\"\" 按照參數設定建置模型 \"\"\"\n",
        "    src_dict, tgt_dict = task.source_dictionary, task.target_dictionary\n",
        "\n",
        "    # 詞嵌入\n",
        "    encoder_embed_tokens = nn.Embedding(len(src_dict), args.encoder_embed_dim, src_dict.pad())\n",
        "    decoder_embed_tokens = nn.Embedding(len(tgt_dict), args.decoder_embed_dim, tgt_dict.pad())\n",
        "    \n",
        "    # 編碼器與解碼器\n",
        "    # TODO: 替換成 TransformerEncoder 和 TransformerDecoder\n",
        "    encoder = RNNEncoder(args, src_dict, encoder_embed_tokens)\n",
        "    decoder = RNNDecoder(args, tgt_dict, decoder_embed_tokens)\n",
        "    \n",
        "    # 序列到序列模型\n",
        "    model = Seq2Seq(args, encoder, decoder)\n",
        "    \n",
        "    # 序列到序列模型的初始化很重要 需要特別處理\n",
        "    def init_params(module):\n",
        "        from fairseq.modules import MultiheadAttention\n",
        "        if isinstance(module, nn.Linear):\n",
        "            module.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            if module.bias is not None:\n",
        "                module.bias.data.zero_()\n",
        "        if isinstance(module, nn.Embedding):\n",
        "            module.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            if module.padding_idx is not None:\n",
        "                module.weight.data[module.padding_idx].zero_()\n",
        "        if isinstance(module, MultiheadAttention):\n",
        "            module.q_proj.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            module.k_proj.weight.data.normal_(mean=0.0, std=0.02)\n",
        "            module.v_proj.weight.data.normal_(mean=0.0, std=0.02)\n",
        "        if isinstance(module, nn.RNNBase):\n",
        "            for name, param in module.named_parameters():\n",
        "                if \"weight\" in name or \"bias\" in name:\n",
        "                    param.data.uniform_(-0.1, 0.1)\n",
        "            \n",
        "    # 初始化模型\n",
        "    model.apply(init_params)\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OCnIeyanpP_I"
      },
      "source": [
        "## 設定模型相關參數\n",
        "參考參數\n",
        "\n",
        "|model|embedding dim|encoder ffn|encoder layers|decoder ffn|decoder layers|\n",
        "|-|-|-|-|-|-|\n",
        "|RNN|256|512|1|1024|1|\n",
        "|Transformer|256|1024|4|1024|4|\n",
        "\n",
        "Strong baseline 用的參數可以參考 [Attention is all you need](#vaswani2017) 的 Table 3 的 transformer-base"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uDFkwQl4pP_I"
      },
      "source": [
        "arch_args = Namespace(\n",
        "    encoder_embed_dim=256,\n",
        "    encoder_ffn_embed_dim=512,\n",
        "    encoder_layers=1,\n",
        "    decoder_embed_dim=256,\n",
        "    decoder_ffn_embed_dim=1024,\n",
        "    decoder_layers=1,\n",
        "    share_decoder_input_output_embed=True,\n",
        "    dropout=0.3,\n",
        ")\n",
        "\n",
        "# # HINT: 補上Transformer用的參數\n",
        "# def add_transformer_args(args):\n",
        "#     args.encoder_attention_heads=4\n",
        "#     args.encoder_normalize_before=True\n",
        "    \n",
        "#     args.decoder_attention_heads=4\n",
        "#     args.decoder_normalize_before=True\n",
        "    \n",
        "#     args.activation_fn=\"relu\"\n",
        "#     args.max_source_positions=1024\n",
        "#     args.max_target_positions=1024\n",
        "    \n",
        "#     # 補上我們沒有設定的Transformer預設參數\n",
        "#     from fairseq.models.transformer import base_architecture \n",
        "#     base_architecture(arch_args)\n",
        "\n",
        "# add_transformer_args(arch_args)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmMR3nPypP_I"
      },
      "source": [
        "if config.use_wandb:\n",
        "    wandb.config.update(vars(arch_args))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vvFKTX2zpP_I",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a81bbc5-e00b-461b-c33f-42b1e4eacb25"
      },
      "source": [
        "model = build_model(arch_args, task)\n",
        "logger.info(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:30:43 | INFO | hw5.seq2seq | Seq2Seq(\n",
            "  (encoder): RNNEncoder(\n",
            "    (embed_tokens): Embedding(8000, 256, padding_idx=1)\n",
            "    (dropout_in_module): Dropout(p=0.3, inplace=False)\n",
            "    (rnn): GRU(256, 512, dropout=0.3, bidirectional=True)\n",
            "    (dropout_out_module): Dropout(p=0.3, inplace=False)\n",
            "  )\n",
            "  (decoder): RNNDecoder(\n",
            "    (embed_tokens): Embedding(8000, 256, padding_idx=1)\n",
            "    (dropout_in_module): Dropout(p=0.3, inplace=False)\n",
            "    (rnn): GRU(256, 1024, dropout=0.3)\n",
            "    (attention): AttentionLayer(\n",
            "      (input_proj): Linear(in_features=256, out_features=1024, bias=False)\n",
            "      (output_proj): Linear(in_features=1280, out_features=256, bias=False)\n",
            "    )\n",
            "    (dropout_out_module): Dropout(p=0.3, inplace=False)\n",
            "    (project_out_dim): Linear(in_features=1024, out_features=256, bias=True)\n",
            "    (output_projection): Linear(in_features=256, out_features=8000, bias=False)\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/rnn.py:65: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.3 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m7NHktqKpP_J"
      },
      "source": [
        "# Optimization 最佳化"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N9uAre01pP_J"
      },
      "source": [
        "## Loss: Label Smoothing Regularization\n",
        "* 讓模型學習輸出較不集中的分佈，防止模型過度自信\n",
        "* 有時候Ground Truth並非唯一答案，所以在算loss時，我們會保留一部份機率給正確答案以外的label\n",
        "* 可以有效防止過度擬合\n",
        "\n",
        "code [source](https://fairseq.readthedocs.io/en/latest/_modules/fairseq/criterions/label_smoothed_cross_entropy.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Ooo0T38pP_J"
      },
      "source": [
        "class LabelSmoothedCrossEntropyCriterion(nn.Module):\n",
        "    def __init__(self, smoothing, ignore_index=None, reduce=True):\n",
        "        super().__init__()\n",
        "        self.smoothing = smoothing\n",
        "        self.ignore_index = ignore_index\n",
        "        self.reduce = reduce\n",
        "    \n",
        "    def forward(self, lprobs, target):\n",
        "        if target.dim() == lprobs.dim() - 1:\n",
        "            target = target.unsqueeze(-1)\n",
        "        # nll: Negative log likelihood，當目標是one-hot時的cross-entropy loss. 以下同 F.nll_loss\n",
        "        nll_loss = -lprobs.gather(dim=-1, index=target)\n",
        "        # 將一部分正確答案的機率分配給其他label 所以當計算cross-entropy時等於把所有label的log prob加起來\n",
        "        smooth_loss = -lprobs.sum(dim=-1, keepdim=True)\n",
        "        if self.ignore_index is not None:\n",
        "            pad_mask = target.eq(self.ignore_index)\n",
        "            nll_loss.masked_fill_(pad_mask, 0.0)\n",
        "            smooth_loss.masked_fill_(pad_mask, 0.0)\n",
        "        else:\n",
        "            nll_loss = nll_loss.squeeze(-1)\n",
        "            smooth_loss = smooth_loss.squeeze(-1)\n",
        "        if self.reduce:\n",
        "            nll_loss = nll_loss.sum()\n",
        "            smooth_loss = smooth_loss.sum()\n",
        "        # 計算cross-entropy時 加入分配給其他label的loss\n",
        "        eps_i = self.smoothing / lprobs.size(-1)\n",
        "        loss = (1.0 - self.smoothing) * nll_loss + eps_i * smooth_loss\n",
        "        return loss\n",
        "\n",
        "# 一般都用0.1效果就很好了\n",
        "criterion = LabelSmoothedCrossEntropyCriterion(\n",
        "    smoothing=0.1,\n",
        "    ignore_index=task.target_dictionary.pad(),\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TmNqHZAbpP_J"
      },
      "source": [
        "## Optimizer: Adam + lr scheduling\n",
        "Inverse square root 排程對於訓練 Transformer 時的穩定性很重要，後來也用在 RNN 上。\n",
        "根據底下公式來更新 learning rate，前期線性增長，後期根據更新步數方根的倒數來遞減。\n",
        "$$lrate = d_{\\text{model}}^{-0.5}\\cdot\\min({step\\_num}^{-0.5},{step\\_num}\\cdot{warmup\\_steps}^{-1.5})$$\n",
        "code [source](https://nlp.seas.harvard.edu/2018/04/03/attention.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-aO0wh75pP_K"
      },
      "source": [
        "class NoamOpt:\n",
        "    \"Optim wrapper that implements rate.\"\n",
        "    def __init__(self, model_size, factor, warmup, optimizer):\n",
        "        self.optimizer = optimizer\n",
        "        self._step = 0\n",
        "        self.warmup = warmup\n",
        "        self.factor = factor\n",
        "        self.model_size = model_size\n",
        "        self._rate = 0\n",
        "    \n",
        "    @property\n",
        "    def param_groups(self):\n",
        "        return self.optimizer.param_groups\n",
        "        \n",
        "    def multiply_grads(self, c):\n",
        "        \"\"\"Multiplies grads by a constant *c*.\"\"\"                \n",
        "        for group in self.param_groups:\n",
        "            for p in group['params']:\n",
        "                if p.grad is not None:\n",
        "                    p.grad.data.mul_(c)\n",
        "        \n",
        "    def step(self):\n",
        "        \"Update parameters and rate\"\n",
        "        self._step += 1\n",
        "        rate = self.rate()\n",
        "        for p in self.param_groups:\n",
        "            p['lr'] = rate\n",
        "        self._rate = rate\n",
        "        self.optimizer.step()\n",
        "        \n",
        "    def rate(self, step = None):\n",
        "        \"Implement `lrate` above\"\n",
        "        if step is None:\n",
        "            step = self._step\n",
        "        return 0 if not step else self.factor * \\\n",
        "            (self.model_size ** (-0.5) *\n",
        "            min(step ** (-0.5), step * self.warmup ** (-1.5)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iZHg8sCipP_K"
      },
      "source": [
        "## 排程視覺化"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2r8d2oNpP_K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "outputId": "9e50afc9-a9be-49e1-8ee8-c58984a16704"
      },
      "source": [
        "optimizer = NoamOpt(\n",
        "    model_size=arch_args.encoder_embed_dim, \n",
        "    factor=config.lr_factor, \n",
        "    warmup=config.lr_warmup, \n",
        "    optimizer=torch.optim.AdamW(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9, weight_decay=0.0001))\n",
        "plt.plot(np.arange(1, 100000), [optimizer.rate(i) for i in range(1, 100000)])\n",
        "plt.legend([f\"{optimizer.model_size}:{optimizer.warmup}\"])\n",
        "None"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD4CAYAAADCb7BPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xV1Znw8d+Te0JuEAKEBEiQYAmgoBHxnWptqYLYEae1LYytOkPrtNWP09p5rcy8Yzu09q11OlVHrbVVi44Vea2X1FKx3i9VMFy8AKLhJoFAQoCcBHI5J3neP/ZKOIRzkpPrSQ7P9/M5n+yz9tprr50D58laa++1RFUxxhhjeiou2hUwxhgzPFkAMcYY0ysWQIwxxvSKBRBjjDG9YgHEGGNMryREuwIDafTo0VpYWBjtahhjzLCyfv36g6qa212+mA4ghYWFlJeXR7saxhgzrIjI7kjyWReWMcaYXrEAYowxplcsgBhjjOmViMZARGQBcCcQD/xWVX/WaX8y8DBwNlALfFVVd7l9y4ClQCtwg6quEZEJLv9YQIH7VfVOl38U8DhQCOwCvqKqh0VEXB0WAseAa1R1Q6+v3BgzbPn9fiorK2lqaop2VYa1lJQUCgoKSExM7NXx3QYQEYkH7gEuAiqBd0SkTFW3BGVbChxW1Skishi4DfiqiJQAi4HpwHjgBRGZCgSA76vqBhHJANaLyF9cmTcDL6rqz0TkZvf+B8AlQLF7nQv8yv00xpxiKisrycjIoLCwEO9vS9NTqkptbS2VlZUUFRX1qoxIurDmABWqukNVW4CVwKJOeRYBK9z2E8A812JYBKxU1WZV3QlUAHNUtaq99aCq9cBWID9EWSuAy4PSH1bP20C2iOT18HqNMTGgqamJnJwcCx59ICLk5OT0qRUXSQDJB/YEva/k+Jf9SXlUNQDUATmRHCsihcBsYK1LGquqVW57P143V6T1QESuFZFyESmvqanp/uqMMcOSBY++6+vvMKqD6CKSDvwB+K6q+jrvV2+u+R7NN6+q96tqqaqW5uZ2+xxMv1n9fhUHG5oH7XzGGBNtkQSQvcCEoPcFLi1kHhFJALLwBtPDHisiiXjB41FVfTIoz4H2rin3s7oH9YiK+iY/33l0A1c9sC7aVTHGDII9e/bw2c9+lpKSEqZPn86dd94JwI9+9CPy8/OZNWsWs2bNYvXq1R3HvPfee5x33nlMnz6dmTNndtl19Itf/AIR4eDBg4A3XnHDDTcwZcoUzjjjDDZsOH7/0IoVKyguLqa4uJgVK1Z0pK9fv56ZM2cyZcoUbrjhBgZk7SdV7fKFN9C+AygCkoB3gemd8lwH3Oe2FwOr3PZ0lz/ZHb8D704uwbsL644Q57sduNlt3wz83G1fCvzZHTsXWNdd3c8++2wdDJWHj+mkHzyrk37w7KCcz5hT3ZYtW6J6/n379un69etVVdXn82lxcbFu3rxZf/jDH+rtt99+Un6/368zZ87UTZs2qarqwYMHNRAIhCz7k08+0YsvvlgnTpyoNTU1qqr6pz/9SRcsWKBtbW361ltv6Zw5c1RVtba2VouKirS2tlYPHTqkRUVFeujQIVVVPeecc/Stt97StrY2XbBgga5evTrk+UL9LoFy7eb7VVW7b4GoN6ZxPbAGb7B7lapuFpHlInKZy/YAkCMiFcCN7osfVd0MrAK2AM8B16lqK/A3wNeBz4nIJvda6Mr6GXCRiHwMfN69B1jtAlAF8BvgO93VfbD4Gv3RroIxZhDl5eVx1llnAZCRkcG0adPYuzd8h8jzzz/PGWecwZlnnglATk4O8fHxIfN+73vf4+c///kJ4xPPPPMMV111FSLC3LlzOXLkCFVVVaxZs4aLLrqIUaNGMXLkSC666CKee+45qqqq8Pl8zJ07FxHhqquu4umnn+7H34AnoudAVHU13hd4cNotQdtNwJfDHHsrcGuntDfwWhKh8tcC80KkK15LZ8ixAGJM9PzHHzezZd9JQ6h9UjI+kx/+7fSI8u7atYuNGzdy7rnn8uabb3L33Xfz8MMPU1payi9+8QtGjhzJRx99hIgwf/58ampqWLx4MTfddBMA3/jGN/jWt75FaWkpzzzzDPn5+R2Bpt3evXuZMOF4D35BQQF79+7tMr2goOCk9P5mT6L3g/qmQMf2sZZAFzmNMbGkoaGBL33pS9xxxx1kZmby7W9/m+3bt7Np0yby8vL4/ve/D0AgEOCNN97g0Ucf5Y033uCpp57ixRdfBOC3v/0tpaWlHDt2jJ/+9KcsX748mpfUIzE9G+9g8TUdb4HsOdTI6eMyolgbY04tkbYU+pvf7+dLX/oSV155JV/84hcBGDt2bMf+b37zm3zhC18AvBbABRdcwOjRowFYuHAhGzZsYN68450t27dvZ+fOnR2tj8rKSs466yzWrVtHfn4+e/Ycf4qhsrKS/Px88vPzeeWVV05Iv/DCC8nPz6eysvKk/P3NWiD9ILgLa3ft0SjWxBgzGFSVpUuXMm3aNG688caO9Kqqqo7tp556ihkzZgAwf/583n//fY4dO0YgEODVV1+lpKTkhDJnzpxJdXU1u3btYteuXRQUFLBhwwbGjRvHZZddxsMPP4yq8vbbb5OVlUVeXh7z58/n+eef5/Dhwxw+fJjnn3+e+fPnk5eXR2ZmJm+//TaqysMPP8yiRZ2f/+47a4H0A19QF9Ynh45FsSbGmMHw5ptv8sgjjzBz5kxmzZoFwE9/+lMee+wxNm3ahIhQWFjIr3/9awBGjhzJjTfeyDnnnIOIsHDhQi699FLgxDGQcBYuXMjq1auZMmUKaWlpPPTQQwCMGjWKf//3f+ecc84B4JZbbmHUqFEA3HvvvVxzzTU0NjZyySWXcMkll/T770F0IO4NHiJKS0t1MBaUuvVPW3jk7d0kxsfxd7PzWb5oxoCf05hT2datW5k2bVq0qxETQv0uRWS9qoaPaI61QPqBrzFAVmoio9OT2V1rLRBjzKnBxkD6ga/JT0ZKIpNy0thjXVjGmFOEBZB+4Gvyk5mSwMRRI9hz+BiB1rZoV8mYmBfL3e+Dpa+/Qwsg/aC+KUBmaiKn5Y7A36o2kG7MAEtJSaG2ttaCSB+oWw8kJSWl12XYGEg/8DX6KcwZwZQx6QBsrznK5Nz0KNfKmNhVUFBAZWUltmRD37SvSNhbFkD6ga8pQEZKAqe5AFJR3cBFJWO7OcoY01uJiYm9XkXP9B/rwuojVcXX6CczNZHMlETGZCRTUd0Q7WoZY8yAswDSR43+VgJtSmaKtyj9abnpbK+xAGKMiX0WQPqofSLFzFSvN3DKmHS2VzfY4J4xJuZZAOmj9nmwMjpaICOobw5QU2/L2xpjYpsFkD5qn4k3M6W9BeLNxGvjIMaYWBdRABGRBSKyTUQqROTmEPuTReRxt3+tiBQG7Vvm0reJyPyg9AdFpFpEPuhU1uNBqxTuEpFNLr1QRBqD9t3X24vuT77G9i4s1wIZMwKAChsHMcbEuG5v4xWReOAe4CKgEnhHRMpUdUtQtqXAYVWdIiKLgduAr4pICd4a6dOB8cALIjLVLWv7O+BuvLXRO6jqV4PO/QugLmj3dlWd1fPLHDjHWyBeABmXmUJmSgJbq+qjWS1jjBlwkbRA5gAVqrpDVVuAlUDnieUXASvc9hPAPPEW9F0ErFTVZlXdibee+RwAVX0NOBTupO74rwCP9eB6Bp2v0yC6iDAtL5OtVf27xKYxxgw1kQSQfGBP0PtKlxYyj6oG8FoNOREeG875wAFV/TgorUhENorIqyJyfqiDRORaESkXkfLBeEq1fRC9vQUCMC0vk23762ltszuxjDGxaygPoi/hxNZHFTBRVWcDNwK/F5HMzgep6v2qWqqqpbm5uQNeSV+Tn6T4OJITjv8qS/IyafS32uqExpiYFkkA2QtMCHpf4NJC5hGRBCALqI3w2JO4Mr4IPN6e5rrBat32emA7MDWC+g8obyLFBLweN0/JeC+u2TiIMSaWRRJA3gGKRaRIRJLwBsXLOuUpA65221cAL6n3JF0ZsNjdpVUEFAPrIjjn54EPVbVjVXgRyXUD+ojIZFfWjgjKGlC+Rv8J3VfgPUwYHydsqaoLc5Qxxgx/3d6FpaoBEbkeWAPEAw+q6mYRWQ6Uq2oZ8ADwiIhU4A2ML3bHbhaRVcAWIABc5+7AQkQeAy4ERotIJfBDVX3AnXYxJw+eXwAsFxE/0AZ8S1XDDsIPlvaJFIOlJMZzWu4Ia4EYY2JaRLPxqupqYHWntFuCtpuAL4c59lbg1hDpS7o43zUh0v4A/CGS+g6m9okUO5uWl8m6nVGPb8YYM2CG8iD6sFDfdHIXFsD08ZlU1TVxsMGmNDHGxCYLIH3kc4PonZ1ZkA3Ae5VHBrtKxhgzKCyA9JGv0d8xkWKwGflZxAls+sQCiDEmNlkA6YPmQCvNgbaOiRSDjUhOYOrYDDZV2p1YxpjYZAGkD46vBXJyCwRg1oRs3t1zxNYGMcbEJAsgfRBqGpNgZ07Ipq7Rz67aY4NZLWOMGRQWQPqgfSLFzs+BtJs1wRtI37Tn8KDVyRhjBosFkD7oaIGE6cIqHpNOamK8DaQbY2KSBZA+6BgDCdOFlRAfx5kTsijfbS0QY0zssQDSBx2LSYV4DqTduUU5bKnydeQ1xphYYQGkD7obRAc4t2gUqlC+y6Y1McbEFgsgfeBr8hMfJ6QlxYfNM3viSBLjhbU2L5YxJsZYAOkDX6M3E2/wWiCdpSbFc2ZBNmt3WAAxxsQWCyB9EG4ixc7mFI3ig711HG0ODEKtjDFmcFgA6YNwEyl2du7kHAJtyoZP7G4sY0zssADSB75GPxnJ3bdASid54yBvVtQOQq2MMWZwRBRARGSBiGwTkQoRuTnE/mQRedztXysihUH7lrn0bSIyPyj9QRGpFpEPOpX1IxHZKyKb3Gthd2VFi6/JH1ELZERyAmdPGsmrH9UMQq2MMWZwdBtA3Drk9wCXACXAEhEp6ZRtKXBYVacAvwRuc8eW4C1POx1YANzbvq458DuXFsovVXWWe62OoKyoqG8KRDQGAnDB1Fy2Vvmo9jUNcK2MMWZwRNICmQNUqOoOVW0BVgKLOuVZBKxw208A88S7NWkRsFJVm1V1J1DhykNVX8NbPz1SYcuKlnDL2Ybymam5ALz28cGBrJIxxgyaSAJIPrAn6H2lSwuZR1UDQB2QE+GxoVwvIu+5bq6RPagHInKtiJSLSHlNzcB1GQVa2zja0hp2IsXOpo3LZHR6Mq9ZN5YxJkYMxUH0XwGnAbOAKuAXPTlYVe9X1VJVLc3NzR2I+gHdz4PVWVyccMHU0bz+cQ2tbbY+iDFm+IskgOwFJgS9L3BpIfOISAKQBdRGeOwJVPWAqraqahvwG453U/W4rIHU3WJSoXxmai6Hj/ltnXRjTEyIJIC8AxSLSJGIJOENZJd1ylMGXO22rwBeUm8ZvjJgsbtLqwgoBtZ1dTIRyQt6+3dA+11aPS5rIHVMpBhhFxbAhVPHkBAnPL/lwEBVyxhjBk23AcSNaVwPrAG2AqtUdbOILBeRy1y2B4AcEakAbgRudsduBlYBW4DngOtUtRVARB4D3gJOF5FKEVnqyvq5iLwvIu8BnwW+111Z0dA+kWJGhF1YAFlpicydnMOaD/bbMrfGmGEvoj+f3a20qzul3RK03QR8OcyxtwK3hkhfEib/17uoR8iyoiGSqdxDmT99LP/+zGYqqhsoHpsxEFUzxphBMRQH0YcFXw8H0dtdVDIOgDWb9/d7nYwxZjBZAOml7pazDWdcVgqzJmSzZrONgxhjhjcLIL3kawogAhnJPevCApg/fRzv761jz6FjA1AzY4wZHBZAesnX6Cc9KYG4uPBrgYTzhTO8G83K3t3X39UyxphBYwGkl+qbAj3uvmo3YVQa5xSO5KmNe+1uLGPMsGUBpJd8Tf6IpzEJ5fLZ+VRUN7B5n68fa2WMMYPHAkgv9WQixVAunZlHYrzw9MaoPUxvjDF9YgGkl3xNgR49hd5ZdloSF54+hrJ399ncWMaYYckCSC9Fuh56V744O5/q+mabodcYMyxZAOmlvnZhAXy+ZCyj05N5dO3ufqqVMcYMHgsgvdDWptQ3960LCyAxPo6vlBbw0ofV7DvS2E+1M8aYwWEBpBcaWgKo9mwixXCWzJmIAo+/s6fbvMYYM5RYAOmF42uB9K0FAt4zIRcU57LynU8ItLb1uTxjjBksFkB6oWMerH5ogQBcee5EDviabZ0QY8ywYgGkF3o7kWI486aNZVJOGve/tsOeTDfGDBsWQHqhfSr3vjyJHiw+TvjGp4vYtOcI63cf7pcyjTFmoEUUQERkgYhsE5EKEbk5xP5kEXnc7V8rIoVB+5a59G0iMj8o/UERqRaRDzqVdbuIfCgi74nIUyKS7dILRaRRRDa51329vei+6u8uLIArzp7AyLRE7n9tR7+VaYwxA6nbACIi8cA9wCVACbBEREo6ZVsKHFbVKcAvgdvcsSV4a6hPBxYA97ryAH7n0jr7CzBDVc8APgKWBe3brqqz3OtbkV1i/6tv6t8uLIDUpHi+PncSf9l6gB01Df1WrjHGDJRIWiBzgApV3aGqLcBKYFGnPIuAFW77CWCeiIhLX6mqzaq6E6hw5aGqrwGHOp9MVZ9367ADvA0U9PCaBlx/d2G1+/p5hSTFx3HvK9v7tVxjjBkIkQSQfCD4IYVKlxYyj/vyrwNyIjy2K/8I/DnofZGIbBSRV0Xk/FAHiMi1IlIuIuU1NQMzRYiv0U9aUjyJ8f07hJSbkczX5k7iyQ2V7Dx4tF/LNsaY/jZkB9FF5N+AAPCoS6oCJqrqbOBG4Pciktn5OFW9X1VLVbU0Nzd3QOrW16ncu/Ktz5xGUkIcd7348YCUb4wx/SWSALIXmBD0vsClhcwjIglAFlAb4bEnEZFrgC8AV6q7r9V1g9W67fXAdmBqBPXvd/VNgX4dQA+Wm5HMVecV8symvVRU21iIMWboiiSAvAMUi0iRiCThDYqXdcpTBlzttq8AXnJf/GXAYneXVhFQDKzr6mQisgC4CbhMVY8Fpee2D8CLyGRXVlRuWfI19X0ixa780wWTSUmM55cvfDRg5zDGmL7qNoC4MY3rgTXAVmCVqm4WkeUicpnL9gCQIyIVeN1LN7tjNwOrgC3Ac8B1qtoKICKPAW8Bp4tIpYgsdWXdDWQAf+l0u+4FwHsisglvoP5bqnrSIPxg8DX2fSLFruSkJ/ONTxfxp/eq7LkQY8yQJbH85HNpaamWl5f3e7mfuf1lzizI5q4ls/u97HZHmwN89j9fYXx2Kk9953/h3dRmjDEDT0TWq2ppd/mG7CD6UFbfFOiXiRS7MiI5gX+Zfzqb9hzhj+9VDei5jDGmNyyA9JCqeotJDdAgerArzipg+vhMbvvzhzS2tA74+YwxpicsgPRQo7+VQJsO6CB6u7g44ZYvlLD3SCN3vWS39RpjhhYLID3kaxyYp9DDOXdyDl8+u4DfvLaDD/f7BuWcxhgTCQsgPdQxD9YgdGG1+9eF08hMTWTZk+/T1ha7Nz0YY4YXCyA95BuAiRS7M3JEEv/n0mls/OQIj67dPWjnNcaYrlgA6aH2LqyBfA4klL+bnc/5xaP56eoP2WXzZBljhgALID3U3gLJGMQuLAAR4fYrziQpIY7vrdpk66cbY6LOAkgPtU/lPtDPgYQyLiuFn1w+g42fHLEp340xUWcBpIcGYjXCnvjbM8ezaNZ47nzxYzZ8YtOcGGOixwJID/ma/CQlxJGSGN995gGyfNEMxmencP2jGzh0tCVq9TDGnNosgPSQN5FidFof7bJSE/nVlWdz8GgL/7xyI612a68xJgosgPRQfZN/0O/ACmVGfhbLL5vO6x8f5E5bfMoYEwUWQHrI1xQgYxCfAenKV8+ZwJfPLuCuFz/mz+/bhIvGmMFlAaSHvIkUo98CAe/W3h9fPoOzJmbzvVWbeHfPkWhXyRhzCrEA0kMDvRphT6UkxnP/VaXkZiSzdEU5e480RrtKxphTREQBREQWiMg2EakQkZtD7E8Wkcfd/rUiUhi0b5lL3yYi84PSHxSRahH5oFNZo0TkLyLysfs50qWLiNzlynpPRM7q7UX3xUCvRtgbo9OTefDqc2gOtPKPD71D3TF/tKtkjDkFdBtA3Drk9wCXACXAEhEp6ZRtKXBYVacAvwRuc8eW4K2hPh1YANzbvq458DuX1tnNwIuqWgy86N7jzl/sXtcCv4rsEvuXN4g+dFog7YrHZnDf185m58GjXPO7dRxtDkS7SsaYGBdJC2QOUKGqO1S1BVgJLOqUZxGwwm0/AcwTbw3WRcBKVW1W1Z1AhSsPVX0NCLWmeXBZK4DLg9IfVs/bQLaI5EVykf2lyd9Kc6BtSHVhBfubKaO5a8ls3qus45sPl9Pkt0WojDEDJ5IAkg/sCXpf6dJC5lHVAFAH5ER4bGdjVbX9lqL9wNge1AMRuVZEykWkvKampptT9Ux9U3QmUuyJBTPGcfsVZ/DX7bVc//uNtARszixjzMAY0oPoqqpAj56SU9X7VbVUVUtzc3P7tT7Rmkixp754VgE/XjSdF7Ye4DuPrreWiDFmQEQSQPYCE4LeF7i0kHlEJAHIAmojPLazA+1dU+5ndQ/qMaDqoziRYk99/bxCfnL5DF7YWs03Hy63NdWNMf0ukgDyDlAsIkUikoQ3KF7WKU8ZcLXbvgJ4ybUeyoDF7i6tIrwB8HXdnC+4rKuBZ4LSr3J3Y80F6oK6ugZFtCdS7KmvzZ3E7VecwZsVB7n6oXU02MC6MaYfdRtA3JjG9cAaYCuwSlU3i8hyEbnMZXsAyBGRCuBG3J1TqroZWAVsAZ4DrlPVVgAReQx4CzhdRCpFZKkr62fARSLyMfB59x5gNbADbyD+N8B3+nTlvRCN1Qj76sulE7hj8WzW7z7MV+57iwO+pmhXyRgTI8RrKMSm0tJSLS8v77fyfr/2E/71qfd5a9nnyMtK7bdyB8OrH9Xwnf9ZT1ZqIg/9wxxOH5cR7SoZY4YoEVmvqqXd5RvSg+hDTX3T8OrCCvaZqbms+tZ5tKpyxa/+yhsfH4x2lYwxw5wFkB7wNfmJjxPSkqK3FkhfTB+fxVPf+RvGZ6dy9UPreOCNncRyC9QYM7AsgPRA+zQm3jOSw9P47FSe+PZ5zPvUGH787Ba++/gmu0PLGNMrFkB6wNfkH/LPgEQiIyWR+752Nv9y8VTK3t3HF3/1V3bXHo12tYwxw4wFkB6obwoMi2dAIhEXJ1z/uWIevOYc9h4+xqV3vcHTGwf1sRpjzDBnAaQHvLVAhn8LJNhnTx/D6n8+n0+Ny+C7j2/ixsc3ddwsYIwxXbEA0gO+IToTb18VjExj5bVz+e7ni3l6014uvesNyneFmufSGGOOswDSA77G2OnC6iwhPo7vfn4qq/7pPFrblC//+i3+44+bOdZiT68bY0KzANID9TEyiN6V0sJRrPneBXx97iQeenMX8+94jb9W2DMjxpiTWQCJUKC1jaMtrTHZhdVZenICyxfN4PFr5xIvwt//di03PfEutQ3N0a6aMWYIsQASoeE0E29/OXdyDn/+5wv4pwsm8+SGvXz2P19hxV93EWi1NUaMMRZAIuYbxtOY9EVqUjzLFk7jue+ez8yCLH5Ytpkv/PcbrNtpg+zGnOosgESovQWSMYRXIxxIU8Zk8D9Lz+XeK8/C1+jnK79+i2sfLqeiuiHaVTPGRIkFkAh1rAUyjKZy728iwsKZebzw/c9w40VTebPiIPPveI1lT75v08QbcwqyABKhU7ULK5S0pARumFfMqzd9lq/PncQT6/fwmdtf5rbnPuTQ0ZZoV88YM0gsgETI13jqDaJ3Z3R6Mj+6bDov3nghF5eM475Xt/Pp217i/67eSk293bFlTKyLKICIyAIR2SYiFSJyc4j9ySLyuNu/VkQKg/Ytc+nbRGR+d2WKyOsissm99onI0y79QhGpC9p3S18uvKfaWyCx/hxIb0zMSeOuJbP5y/cu4OKSsfzm9R2c//OXWP7HLda1ZUwM6/bPaRGJB+4BLgIqgXdEpExVtwRlWwocVtUpIrIYuA34qoiU4K2hPh0YD7wgIlPdMSHLVNXzg879B46viQ7wuqp+obcX2xe+pgAikJFsLZBwpozJ4I7Fs7lhXjH3vLydFW/t4n/e3s1ls8az9NNFTMvLjHYVjTH9KJIWyBygQlV3qGoLsBJY1CnPImCF234CmCfeohmLgJWq2qyqO/HWM58TSZkikgl8Dni6d5fWv3yNftKTE4iLG75rgQyWybnp/OIrZ/Ly9y9kyZwJ/Om9Ki6583Wu/O3bvPxhNW1ttoiVMbEgkgCSD+wJel/p0kLmUdUAUAfkdHFsJGVeDryoqr6gtPNE5F0R+bOITA9VWRG5VkTKRaS8pqYmgsuLTKxOpDiQJuak8R+LZvDWss/xgwWfoqK6gX/43Ttc9MtXWfHXXR3dgsaY4WkoD6IvAR4Ler8BmKSqZwL/TZiWiarer6qlqlqam5vbb5XxNQZO2WdA+io7LYlvX3gar9/0Oe746ixGJCfww7LNzLn1BW564l027TliS+saMwxF8o24F5gQ9L7ApYXKUykiCUAWUNvNsWHLFJHReN1cf9eeFtwSUdXVInKviIxW1UGZ6a++yX9KPwPSH5IS4rh8dj6Xz87nvcoj/H7tJ5S9u49V5ZWU5GXy9+dO5LJZ462lZ8wwEUkL5B2gWESKRCQJb1C8rFOeMuBqt30F8JJ6f1KWAYvdXVpFQDGwLoIyrwCeVdWOW3hEZJwbV0FE5ri61/bscnvP1xSwL7Z+dEZBNj/70hms/dd5/PjyGSjwf57+gHN+8gLX/34DL314AL/NuWXMkNZtC0RVAyJyPbAGiAceVNXNIrIcKFfVMuAB4BERqQAO4QUEXL5VwBYgAFynqq0AocoMOu1i4GedqnIF8G0RCQCNwGIdxH4PX6OfaXkZg3W6U0ZGSiJfnzuJr507kXcr63hyQyV/fHcfz75Xxej0JC47M58vnpXP9PGZuL8fjDFDhMRy33NpaamWl5f3S1kzf7SGL51VwI8uCzl2b/pRS6CNV7ZV89TGvby4tZqW1mFdcncAABU3SURBVDZOyx3BpTPzWHhGHqePzbBgYswAEpH1qlraXT4bFY5AW5vS0Bwg0wbRB0VSQhwXTx/HxdPHUXfMz7Pv7+PZd6u4++UK7nqpgsm5I1g4I4+FM/OYlmfBxJhosW/ECDS0BFA9tSdSjJastESuPHcSV547iZr6Zp7fsp/V71dx7ysV3P1yBYU5acyfMY55nxrLWROzSYgfyjcWGhNbLIBEoGMmXhtEj6rcjOSOYFLb0MzzWw6w+v0qHnh9J79+dQfZaYlcODWXedPGcsHUXLIs4BszoCyARMAmUhx6ctKTWTJnIkvmTMTX5Of1jw7y4ocHeGVbDU9v2kdCnHBO4SjmTRvD+cW5TB2bbl1dxvQz+0aMQL1NpDikZaYkcukZeVx6Rh6tbcqmPYd5cWs1L26t5id/2gpsZUxGMp+eMppPF4/m01NGMyYzJdrVNmbYswASAV/7eugWQIa8+Djh7EmjOHvSKG5a8Cn2HWnkjY8P8nrFQV75qIYnN3rPq54+NsMLJsWjOadwFOk2SaYxPWb/ayJwfDVC+3UNN+OzU/nKORP4yjkTaGtTtlT5eP3jg7xRUcMjb+/mgTd2EicwIz+LOYWjmFPkvbLTkqJddWOGPPtGjICtRhgb4uKEGflZzMjP4tsXnkaTv5XyXYdZt7OWt3ce4uG3d/PbN3YC8KlxGR3BZE7hKOvyMiYECyARqHddWOn2HEhMSUmM7+jGAmjyt/JeZR1rd9SybtchnlhfycNv7QYgPzuV2ROzmT1xJLMnZjN9fCbJCfHRrL4xUWffiBHwNfpJS4on0Z4xiGkpifEdrQ4Af2sbm/f5eGfnITbtOcKG3Yd59r0qAJLi4ygZn8lZLqDMnphNfnaq3ellTikWQCJga4GcmhLj45g1IZtZE7I70vbXNbFpz2E2fnKEjZ8c4ffrdvPgm163V86IJKbnZzFjfCYzXVdZwUgLKiZ2WQCJgK8xYAPoBoBxWSksyMpjwYw8wGulbNtfz8ZPDvP+3jo+2Ovj/td2EHCrLmalJjIjP5MZ47M6xl8mjUqzlS1NTLBvxQjUN/vtGRATUmJ8XEdgaNfkb+WjA/UdAWXzvjoeenMXLW56+hFJ8Uwdl8GnxmXwqXGZnO627c4vM9xYAImArzHA6HT7z20ik5IYzxkF2ZxRcLzrqyXQxsfV9Wx2AeXD/fX8+YP9PLbu+MrOYzOTOX1cpgssGZw+LoMpY9JtsN4MWRZAIuBr8jM5d0S0q2GGsaSEOKaPz2L6+CzaF+NUVarrm/lwfz3b9vv4sKqeD/fX87vttR2tlfg4YdKoNCbnpjNlTDqn5Y7wfo5Jt3E5E3UWQCLga7RBdNP/RISxmSmMzUzhM1NzO9IDrW3sqj3K1qp6tu2vZ3tNAxXVDbz6UTX+1uPr94zJSOa0EwKL12IZm5lsA/dmUEQUQERkAXAn3uqBv1XVn3Xanww8DJyNt8zsV1V1l9u3DFgKtAI3qOqarsoUkd8BnwHqXPHXqOomt5ztncBC4JhL39C7y46cqnrL2doguhkkCfFxLhhk8LdnHk8PtLbxyaFjbK85SkV1Q0dgeXrjXuqbAx350pLimZQzgsKctI6fhaNHUJgzgjEZyTaAb/pNt9+KIhIP3ANcBFQC74hImapuCcq2FDisqlNEZDFwG/BVESnBW552OjAeeEFEprpjuirzf6vqE52qcgnemurFwLnAr9zPAdXob6W1TW0Q3URdQnwck3PTmZybzkUlYzvSVZWa+mYqqhuoqGlg58Gj7K49xrYD9byw9cAJrZaUxDgmjRrBpKCgUpiTxqTRIxiXmUK8BRfTA5H8WT0HqFDVHQAishJYhLfOebtFwI/c9hPA3a7FsAhYqarNwE63Zvocl6+7MjtbBDzs1kF/W0SyRSRPVasiuIZe65jK3QKIGaJEhDGZKYzJTOF/TRl9wr5AaxtVdU3sqj3Krtpj7D7o/dxx8CivbKvpGGsBSIwXxmenUjAylYLsNCaMSqVgZBoFI1OZMCqN3HRrvZgTRRJA8oE9Qe8rOfkv/448qhoQkTogx6W/3enYfLfdVZm3isgtwIvAzS4AhapHPnBCABGRa4FrASZOnBjB5XWtYx4s68Iyw1BCfBwTRqUxYVQa5xefuK+1Tdnva2L3waPsrD1K5eFGKg83sufQMV78sJqDDc0n5E+KjyN/pAswQYGlYGQq47NSyc1IthbMKWYofisuA/YDScD9wA+A5ZEerKr3u+MoLS3VbrJ3y1YjNLEqPk7Iz04lPzv1pJYLQGNLK3uPHGOPCyyVh49Recj7+fy+/dQebTkhf0Kcd1NAXlYK47JSGJ+dSl5WCnlZ7md2CqNHWCsmlkQSQPbSft+hp8ClhcpTKSIJQBbeYHpXx4ZMD+qSahaRh4B/6UE9+l37RIoZNpGiOcWkJsV3DOaHcrQ5wN4jXoulqq6JqrpGqo40sa+ukQ/21vH8lgO0BNpOOCYx3gsy47NSycs+HlzGZXl3o43JSCY3I9nmnRsmIvlWfAcoFpEivC/sxcDfd8pTBlwNvAVcAbykqioiZcDvReS/8AbRi4F1gIQrs31cw42hXA58EHSO6914yblA3UCPf0BwF5a1QIwJNiI5galjM5g6NnSAUVUOHW1xwcULMPuOuEBT18SGTw6zv67qhEF+ABFvXrHcjBTGZiYzJiO5I7iMcT/HZqZYoBkCug0gbkzjemAN3i23D6rqZhFZDpSrahnwAPCIGyQ/hBcQcPlW4Q2OB4DrVLUVIFSZ7pSPikguXpDZBHzLpa/Gu4W3Au823n/o89VHwLqwjOkdESEnPZmc9OQTpnoJ1tamHDzazIG6Zqrrmzjga+aAr4nq+maq3c8t+3wcbGimLUSHdM6IpKCgksyYjBRy0pMYnZ7M6PRkcjOSyBmRTHZaoj0bMwDEu6kpNpWWlmp5eXmfyrjn5QpuX7OND3+8gJREm1LCmGhobVNqG5o54DseaDp+ukBzwNcUNtAkxAk56V4wGZ2RzOiOIHM82OSkJ5GbnsyoEUkknOItGxFZr6ql3eWzjv1u+Jr8JCXEWfAwJori447fquwNsYbW2qYcOdbCwYYWDjY0u1cLtZ22t1c3UNPQfNIYTbuRaYkdQSVnRDIjRyQyKi2JkSOSGDUiiZFp7ueIJEalJZGadGp+P1gA6YavMWDdV8YME/Fxx7vNTif02Ew7VaWhOXBCgKkJDjb1LdQebWbbgXoOHW3hyLGWkK0b8B7QDA4woYLMyBGJ3r60JLLTkkhKGP6tHAsg3fA1+e0ZEGNikIiQkZJIRkoiRaO7nyy1tU3xNfo5dKyFw0dbOHS0hcPHWjh01O9+eum1R1v45NAxDh1t6biLM5QRSfFkpSaSlZZEdmoi2WmJ7n0i2alJZLm0bJfmvU9iRFL8kBnPsW/GbthEisYY8Fo3I12Lgtzu84O34NjhYy0cPuoPCjjeq67Rz5Fjfuoa/dQ1tlBR3cCRRj91x/wnzBDQWUKcBAWa44ElKygIZaclUjQ6/YTVNAeCBZBu1DcF7BkQY0yvJMbHMSYjhTEZKREfo6o0+ds40ng8yBw55sfX6OdIY4v3vtEFnmN+ahqaqahp4Mgx/wktnr89czz/vWT2QFxWB/tm7IavyU/+yNRoV8MYc4oQEVKT4klNSiUvq2ffPYHWNuqbAhxp9JMwCE/8WwDphg2iG2OGi4T4uOPdbINg+N8GMMBsEN0YY0KzANKFJn8rLYE2a4EYY0wIFkC60D4glWmD6MYYcxILIF2wiRSNMSY8CyBdsIkUjTEmPAsgXejowrJBdGOMOYkFkC60d2FlWAvEGGNOYgGkC77G9kF0CyDGGNOZBZAuHB9Ety4sY4zpLKIAIiILRGSbiFSIyM0h9ieLyONu/1oRKQzat8ylbxOR+d2VKSKPuvQPRORBEUl06ReKSJ2IbHKvW/py4ZHwuekAUm0tEGOMOUm3AURE4oF7gEuAEmCJiJR0yrYUOKyqU4BfAre5Y0vwlredDiwA7hWR+G7KfBT4FDATSAW+EXSe11V1lnst780F90T7RIpDZepkY4wZSiJpgcwBKlR1h6q2ACuBRZ3yLAJWuO0ngHnifesuAlaqarOq7sRbz3xOV2Wq6mp1gHVAQd8usfe8aUxs/MMYY0KJJIDkA3uC3le6tJB5VDUA1AE5XRzbbZmu6+rrwHNByeeJyLsi8mcRmR6qsiJyrYiUi0h5TU1NBJcXnq0FYowx4Q3lQfR7gddU9XX3fgMwSVXPBP4beDrUQap6v6qWqmppbm6Eq76E4WsK2AC6McaEEUkA2QtMCHpf4NJC5hGRBLxV72u7OLbLMkXkh3hrft3YnqaqPlVtcNurgUQRGR1B/XutvslPRrK1QIwxJpRIAsg7QLGIFIlIEt6geFmnPGXA1W77CuAlN4ZRBix2d2kVAcV44xphyxSRbwDzgSWq2rGuo4iMc+MqiMgcV/fa3lx0pHyN1gIxxphwuv12VNWAiFwPrAHigQdVdbOILAfKVbUMeAB4REQqgEN4AQGXbxWwBQgA16lqK0CoMt0p7wN2A2+5ePGku+PqCuDbIhIAGoHFLkgNGF+TjYEYY0w4Ef157bqMVndKuyVouwn4cphjbwVujaRMlx6yTqp6N3B3JPXtD/7WNo61tNpdWMYYE8ZQHkSPqgZbC8QYY7pkASQMm0jRGGO6ZgEkjI6JFK0LyxhjQrIAEkbHRIrWhWWMMSFZAAmj3pazNcaYLlkACaO9CyvDWiDGGBOSBZAwfNYCMcaYLlkACcPX6EcE0pOsBWKMMaFYAAnD1xQgIzmBuDhbC8QYY0KxABKGr8lvz4AYY0wXLICE4U2kaAHEGGPCsQAShjeRoo1/GGNMOBZAwqhvshaIMcZ0xQJIGL5Gvz0DYowxXbAAEoatBWKMMV2zABJCW5vS0GxdWMYY05WIAoiILBCRbSJSISI3h9ifLCKPu/1rRaQwaN8yl75NROZ3V6Zb5natS3/cLXnb5Tn6W0NLAFWbSNEYY7rSbQARkXjgHuASoARYIiIlnbItBQ6r6hTgl8Bt7tgSvOVtpwMLgHtFJL6bMm8DfunKOuzKDnuOgeBrtGlMjDGmO5G0QOYAFaq6Q1VbgJXAok55FgEr3PYTwDzxFjRfBKxU1WZV3QlUuPJClumO+ZwrA1fm5d2co991rAViLRBjjAkrkgCSD+wJel/p0kLmUdUAUAfkdHFsuPQc4Igro/O5wp3jBCJyrYiUi0h5TU1NBJd3spTEOC6dmUfByLReHW+MMaeCmBtEV9X7VbVUVUtzc3N7Vcbk3HTuufIsZuRn9XPtjDEmdkQSQPYCE4LeF7i0kHlEJAHIAmq7ODZcei2Q7crofK5w5zDGGBMFkQSQd4Bid3dUEt6geFmnPGXA1W77CuAlVVWXvtjdQVUEFAPrwpXpjnnZlYEr85luzmGMMSYKuh0lVtWAiFwPrAHigQdVdbOILAfKVbUMeAB4REQqgEN4AQGXbxWwBQgA16lqK0CoMt0pfwCsFJGfABtd2YQ7hzHGmOiQWP4jvrS0VMvLy6NdDWOMGVZEZL2qlnaXL+YG0Y0xxgwOCyDGGGN6xQKIMcaYXrEAYowxpldiehBdRGqA3X0oYjRwsJ+qMxycatcLds2nCrvmnpmkqt0+iR3TAaSvRKQ8kjsRYsWpdr1g13yqsGseGNaFZYwxplcsgBhjjOkVCyBduz/aFRhkp9r1gl3zqcKueQDYGIgxxphesRaIMcaYXrEAYowxplcsgIQgIgtEZJuIVIjIzdGuT0+IyAQReVlEtojIZhH5Z5c+SkT+IiIfu58jXbqIyF3uWt8TkbOCyrra5f9YRK4OSj9bRN53x9w1UEsL95SIxIvIRhF51r0vEpG1rp6Pu6UDcMsLPO7S14pIYVAZy1z6NhGZH5Q+5P5NiEi2iDwhIh+KyFYROS/WP2cR+Z77d/2BiDwmIimx9jmLyIMiUi0iHwSlDfjnGu4cXVJVewW98KaX3w5MBpKAd4GSaNerB/XPA85y2xnAR0AJ8HPgZpd+M3Cb214I/BkQYC6w1qWPAna4nyPd9ki3b53LK+7YS6J93a5eNwK/B55171cBi932fcC33fZ3gPvc9mLgcbdd4j7vZKDI/TuIH6r/JoAVwDfcdhKQHcufM96y1juB1KDP95pY+5yBC4CzgA+C0gb8cw13ji7rGu3/BEPtBZwHrAl6vwxYFu169eF6ngEuArYBeS4tD9jmtn8NLAnKv83tXwL8Oij91y4tD/gwKP2EfFG8zgLgReBzwLPuP8dBIKHz54q3Ds15bjvB5ZPOn3V7vqH4bwJvRc6duBthOn9+sfg54wWQPe5LMcF9zvNj8XMGCjkxgAz45xruHF29rAvrZO3/SNtVurRhxzXZZwNrgbGqWuV27QfGuu1w19tVemWI9Gi7A7gJaHPvc4Ajqhpw74Pr2XFtbn+dy9/T30U0FQE1wEOu2+63IjKCGP6cVXUv8J/AJ0AV3ue2ntj+nNsNxuca7hxhWQCJUSKSDvwB+K6q+oL3qfcnRszcvy0iXwCqVXV9tOsyiBLwujl+paqzgaN43Q4dYvBzHgkswgue44ERwIKoVioKBuNzjfQcFkBOtheYEPS+wKUNGyKSiBc8HlXVJ13yARHJc/vzgGqXHu56u0ovCJEeTX8DXCYiu4CVeN1YdwLZItK+bHNwPTuuze3PAmrp+e8imiqBSlVd694/gRdQYvlz/jywU1VrVNUPPIn32cfy59xuMD7XcOcIywLIyd4Bit2dHUl4g29lUa5TxNwdFQ8AW1X1v4J2lQHtd2JcjTc20p5+lbubYy5Q55qxa4CLRWSk+8vvYrz+4SrAJyJz3bmuCiorKlR1maoWqGoh3uf1kqpeCbwMXOGydb7m9t/FFS6/uvTF7u6dIqAYb8BxyP2bUNX9wB4ROd0lzQO2EMOfM17X1VwRSXN1ar/mmP2cgwzG5xruHOFFc1BsqL7w7mz4CO+OjH+Ldn16WPdP4zU93wM2uddCvL7fF4GPgReAUS6/APe4a30fKA0q6x+BCvf6h6D0UuADd8zddBrIjfL1X8jxu7Am430xVAD/D0h26SnufYXbPzno+H9z17WNoLuOhuK/CWAWUO4+66fx7raJ6c8Z+A/gQ1evR/DupIqpzxl4DG+Mx4/X0lw6GJ9ruHN09bKpTIwxxvSKdWEZY4zpFQsgxhhjesUCiDHGmF6xAGKMMaZXLIAYY4zpFQsgxhhjesUCiDHGmF75/wfZTRHPkst0AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dqp06bixpP_K"
      },
      "source": [
        "# 訓練步驟"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uy3p4-uypP_L"
      },
      "source": [
        "## Training 訓練"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RlnrxiFPpP_L"
      },
      "source": [
        "from fairseq.data import iterators\n",
        "from torch.cuda.amp import GradScaler, autocast\n",
        "\n",
        "def train_one_epoch(epoch_itr, model, task, criterion, optimizer, accum_steps=1):\n",
        "    itr = epoch_itr.next_epoch_itr(shuffle=True)\n",
        "    itr = iterators.GroupedIterator(itr, accum_steps) # 梯度累積: 每 accum_steps 個 sample 更新一次\n",
        "    \n",
        "    stats = {\"loss\": []}\n",
        "    scaler = GradScaler() # 混和精度訓練 automatic mixed precision (amp) \n",
        "    \n",
        "    model.train()\n",
        "    progress = tqdm.tqdm(itr, desc=f\"train epoch {epoch_itr.epoch}\", leave=False)\n",
        "    for samples in progress:\n",
        "        model.zero_grad()\n",
        "        accum_loss = 0\n",
        "        sample_size = 0\n",
        "        # 梯度累積: 每 accum_steps 個 sample 更新一次\n",
        "        for i, sample in enumerate(samples):\n",
        "            if i == 1:\n",
        "                # emptying the CUDA cache after the first step can reduce the chance of OOM\n",
        "                torch.cuda.empty_cache()\n",
        "\n",
        "            sample = utils.move_to_cuda(sample, device=device)\n",
        "            target = sample[\"target\"]\n",
        "            sample_size_i = sample[\"ntokens\"]\n",
        "            sample_size += sample_size_i\n",
        "            \n",
        "            # 混和精度訓練 \n",
        "            with autocast():\n",
        "                net_output = model.forward(**sample[\"net_input\"])\n",
        "                lprobs = F.log_softmax(net_output[0], -1)            \n",
        "                loss = criterion(lprobs.view(-1, lprobs.size(-1)), target.view(-1))\n",
        "                \n",
        "                # logging\n",
        "                accum_loss += loss.item()\n",
        "                # back-prop\n",
        "                scaler.scale(loss).backward()                \n",
        "        \n",
        "        scaler.unscale_(optimizer)\n",
        "        optimizer.multiply_grads(1 / (sample_size or 1.0)) # (sample_size or 1.0) handles the case of a zero gradient\n",
        "        gnorm = nn.utils.clip_grad_norm_(model.parameters(), config.clip_norm) # 梯度裁剪 防止梯度爆炸\n",
        "        \n",
        "        scaler.step(optimizer)\n",
        "        scaler.update()\n",
        "        \n",
        "        # logging\n",
        "        loss_print = accum_loss/sample_size\n",
        "        stats[\"loss\"].append(loss_print)\n",
        "        progress.set_postfix(loss=loss_print)\n",
        "        if config.use_wandb:\n",
        "            wandb.log({\n",
        "                \"train/loss\": loss_print,\n",
        "                \"train/grad_norm\": gnorm.item(),\n",
        "                \"train/lr\": optimizer.rate(),\n",
        "                \"train/sample_size\": sample_size,\n",
        "            })\n",
        "        \n",
        "    loss_print = np.mean(stats[\"loss\"])\n",
        "    logger.info(f\"training loss: {loss_print:.4f}\")\n",
        "    return stats"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I3QNnLRNpP_M"
      },
      "source": [
        "## Validation & Inference 檢驗和推論\n",
        "為防止訓練發生過度擬合，每過一段時間要做一次檢測，計算模型在未看過的資料上的表現。\n",
        "- 過程基本上和training一樣，另外加上 inference\n",
        "- 檢驗完畢可順便儲存模型參數\n",
        "\n",
        "單看 validation loss，我們很難知道模型真實的效能\n",
        "- 直接用當前模型去生成翻譯結果 (hypothesis)，再和正確答案 (reference) 計算 BLEU score\n",
        "- 也可用肉眼看翻譯結果的好壞\n",
        "- 我們用 fairseq 寫好的 sequence generator 來進行 beam search 生成翻譯結果"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLsINtUPpP_M"
      },
      "source": [
        "# fairseq 的 beam search generator\n",
        "# 給定模型和輸入序列，用 beam search 生成翻譯結果\n",
        "sequence_generator = task.build_generator([model], config)\n",
        "\n",
        "def decode(toks, dictionary):\n",
        "    # 從 Tensor 轉成人看得懂的句子\n",
        "    s = dictionary.string(\n",
        "        toks.int().cpu(),\n",
        "        config.post_process,\n",
        "    )\n",
        "    return s if s else \"<unk>\"\n",
        "\n",
        "def inference_step(sample, model):\n",
        "    gen_out = sequence_generator.generate([model], sample)\n",
        "    srcs = []\n",
        "    hyps = []\n",
        "    refs = []\n",
        "    for i in range(len(gen_out)):\n",
        "        # 對於每個 sample, 收集輸入，輸出和參考答案，稍後計算 BLEU\n",
        "        srcs.append(decode(\n",
        "            utils.strip_pad(sample[\"net_input\"][\"src_tokens\"][i], task.source_dictionary.pad()), \n",
        "            task.source_dictionary,\n",
        "        ))\n",
        "        hyps.append(decode(\n",
        "            gen_out[i][0][\"tokens\"], # 0 代表取出 beam 內分數第一的輸出結果\n",
        "            task.target_dictionary,\n",
        "        ))\n",
        "        refs.append(decode(\n",
        "            utils.strip_pad(sample[\"target\"][i], task.target_dictionary.pad()), \n",
        "            task.target_dictionary,\n",
        "        ))\n",
        "    return srcs, hyps, refs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEXq75qppP_N"
      },
      "source": [
        "import shutil\n",
        "import sacrebleu\n",
        "\n",
        "def validate(model, task, criterion, log_to_wandb=True):\n",
        "    logger.info('begin validation')\n",
        "    itr = load_data_iterator(task, \"valid\", 1, config.max_tokens, config.num_workers).next_epoch_itr(shuffle=False)\n",
        "    \n",
        "    stats = {\"loss\":[], \"bleu\": 0, \"srcs\":[], \"hyps\":[], \"refs\":[]}\n",
        "    srcs = []\n",
        "    hyps = []\n",
        "    refs = []\n",
        "    \n",
        "    model.eval()\n",
        "    progress = tqdm.tqdm(itr, desc=f\"validation\", leave=False)\n",
        "    with torch.no_grad():\n",
        "        for i, sample in enumerate(progress):\n",
        "            # validation loss\n",
        "            sample = utils.move_to_cuda(sample, device=device)\n",
        "            net_output = model.forward(**sample[\"net_input\"])\n",
        "\n",
        "            lprobs = F.log_softmax(net_output[0], -1)\n",
        "            target = sample[\"target\"]\n",
        "            sample_size = sample[\"ntokens\"]\n",
        "            loss = criterion(lprobs.view(-1, lprobs.size(-1)), target.view(-1)) / sample_size\n",
        "            progress.set_postfix(valid_loss=loss.item())\n",
        "            stats[\"loss\"].append(loss)\n",
        "            \n",
        "            # 進行推論\n",
        "            s, h, r = inference_step(sample, model)\n",
        "            srcs.extend(s)\n",
        "            hyps.extend(h)\n",
        "            refs.extend(r)\n",
        "            \n",
        "    tok = 'zh' if task.cfg.target_lang == 'zh' else '13a'\n",
        "    stats[\"loss\"] = torch.stack(stats[\"loss\"]).mean().item()\n",
        "    stats[\"bleu\"] = sacrebleu.corpus_bleu(hyps, [refs], tokenize=tok) # 計算BLEU score\n",
        "    stats[\"srcs\"] = srcs\n",
        "    stats[\"hyps\"] = hyps\n",
        "    stats[\"refs\"] = refs\n",
        "    \n",
        "    if config.use_wandb and log_to_wandb:\n",
        "        wandb.log({\n",
        "            \"valid/loss\": stats[\"loss\"],\n",
        "            \"valid/bleu\": stats[\"bleu\"].score,\n",
        "        }, commit=False)\n",
        "    \n",
        "    showid = np.random.randint(len(hyps))\n",
        "    logger.info(\"example source: \" + srcs[showid])\n",
        "    logger.info(\"example hypothesis: \" + hyps[showid])\n",
        "    logger.info(\"example reference: \" + refs[showid])\n",
        "    \n",
        "    # show bleu results\n",
        "    logger.info(f\"validation loss:\\t{stats['loss']:.4f}\")\n",
        "    logger.info(stats[\"bleu\"].format())\n",
        "    return stats"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ifHUFMbGpP_N"
      },
      "source": [
        "# 儲存及載入模型參數"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bwCGWvrPpP_N"
      },
      "source": [
        "def validate_and_save(model, task, criterion, optimizer, epoch, save=True):   \n",
        "    stats = validate(model, task, criterion)\n",
        "    bleu = stats['bleu']\n",
        "    loss = stats['loss']\n",
        "    if save:\n",
        "        # save epoch checkpoints\n",
        "        savedir = Path(config.savedir).absolute()\n",
        "        savedir.mkdir(parents=True, exist_ok=True)\n",
        "        \n",
        "        check = {\n",
        "            \"model\": model.state_dict(),\n",
        "            \"stats\": {\"bleu\": bleu.score, \"loss\": loss},\n",
        "            \"optim\": {\"step\": optimizer._step}\n",
        "        }\n",
        "        torch.save(check, savedir/f\"checkpoint{epoch}.pt\")\n",
        "        shutil.copy(savedir/f\"checkpoint{epoch}.pt\", savedir/f\"checkpoint_last.pt\")\n",
        "        logger.info(f\"saved epoch checkpoint: {savedir}/checkpoint{epoch}.pt\")\n",
        "    \n",
        "        # save epoch samples\n",
        "        with open(savedir/f\"samples{epoch}.{config.source_lang}-{config.target_lang}.txt\", \"w\") as f:\n",
        "            for s, h in zip(stats[\"srcs\"], stats[\"hyps\"]):\n",
        "                f.write(f\"{s}\\t{h}\\n\")\n",
        "\n",
        "        # get best valid bleu    \n",
        "        if getattr(validate_and_save, \"best_bleu\", 0) < bleu.score:\n",
        "            validate_and_save.best_bleu = bleu.score\n",
        "            torch.save(check, savedir/f\"checkpoint_best.pt\")\n",
        "            \n",
        "        del_file = savedir / f\"checkpoint{epoch - config.keep_last_epochs}.pt\"\n",
        "        if del_file.exists():\n",
        "            del_file.unlink()\n",
        "    \n",
        "    return stats\n",
        "\n",
        "def try_load_checkpoint(model, optimizer=None, name=None):\n",
        "    name = name if name else \"checkpoint_last.pt\"\n",
        "    checkpath = Path(config.savedir)/name\n",
        "    if checkpath.exists():\n",
        "        check = torch.load(checkpath)\n",
        "        model.load_state_dict(check[\"model\"])\n",
        "        stats = check[\"stats\"]\n",
        "        step = \"unknown\"\n",
        "        if optimizer != None:\n",
        "            optimizer._step = step = check[\"optim\"][\"step\"]\n",
        "        logger.info(f\"loaded checkpoint {checkpath}: step={step} loss={stats['loss']} bleu={stats['bleu']}\")\n",
        "    else:\n",
        "        logger.info(f\"no checkpoints found at {checkpath}!\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tYkO9LltpP_O"
      },
      "source": [
        "# 主程式\n",
        "## 訓練迴圈"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4p7oPThZpP_O"
      },
      "source": [
        "model = model.to(device=device)\n",
        "criterion = criterion.to(device=device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvNfLwAapP_O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b205137a-4c6d-46be-a610-d8ca03f79ec6"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon Aug  2 08:32:51 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 470.42.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   38C    P0    26W /  70W |   1138MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gytSmdH6pP_O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f502f83-5470-4b2e-b8de-0d5fba9cb418"
      },
      "source": [
        "logger.info(\"task: {}\".format(task.__class__.__name__))\n",
        "logger.info(\"encoder: {}\".format(model.encoder.__class__.__name__))\n",
        "logger.info(\"decoder: {}\".format(model.decoder.__class__.__name__))\n",
        "logger.info(\"criterion: {}\".format(criterion.__class__.__name__))\n",
        "logger.info(\"optimizer: {}\".format(optimizer.__class__.__name__))\n",
        "logger.info(\n",
        "    \"num. model params: {:,} (num. trained: {:,})\".format(\n",
        "        sum(p.numel() for p in model.parameters()),\n",
        "        sum(p.numel() for p in model.parameters() if p.requires_grad),\n",
        "    )\n",
        ")\n",
        "logger.info(f\"max tokens per batch = {config.max_tokens}, accumulate steps = {config.accum_steps}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | task: TranslationTask\n",
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | encoder: RNNEncoder\n",
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | decoder: RNNDecoder\n",
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | criterion: LabelSmoothedCrossEntropyCriterion\n",
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | optimizer: NoamOpt\n",
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | num. model params: 11,251,968 (num. trained: 11,251,968)\n",
            "2021-08-02 08:33:11 | INFO | hw5.seq2seq | max tokens per batch = 8192, accumulate steps = 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "VJtMvf9TpP_P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "bf7ce63a840349ae9bb72dced9b02515",
            "8520977de3cb46ef8222deb614432fac",
            "460407d798934975a8aab674701bd0a4",
            "9ae42508de4c446b940d7231b6c9240a",
            "75c42e9365c84fd5b10adc531cb107e3",
            "056c2b8281154b01bc48f401d35e6dd2",
            "850f0de13e5d41cbaecb335b4952b44d",
            "ff4ab00a6d8a4f60af13a6ac879acfd5",
            "d58e36f69fb849109fb31716510ae80a",
            "36293fde6f084386865516c1c6593e06",
            "375fb18eeb964560a7e5c316043f7f53",
            "3c55a5be326d428baba848e493f25fad",
            "921e0c6ad90846b3ab36954e84de1e08",
            "9f6fb85f005f4ac1822671190e11cc50",
            "229667fc0dc14a7c9e8320628203abd9",
            "0b5cc9f6a6f34599bc4dab5063f901f1",
            "fa08453f6ff34f0187434138e43c04b2",
            "2a04d4add9074aa7becf5d9dcdb3d3c9",
            "fd9034e2397741249fd3d571925e5ff9",
            "6b913313c5774239b84def9f6ebf165d",
            "ab84246147e74efcbe1c2dbf402b9adb",
            "1bd0000f9f334877bff2138630ca8385",
            "850a486fa62b495cb9b2f51a533451b1",
            "25e01bad048d4b43acd0f9edfc319997",
            "c19f12cf4b4f41a3a0d32dfb777fab3c",
            "e2472c50cfc5438b9d273e9ab8041ab2",
            "6ec2e45904b44e7d9d87fd9b2557ece1",
            "2c6d05ba22cf433eb97d9c0b3d849287",
            "21e1789d219b4b3bad789d6465463971",
            "8e2ec4a1728741cc91e5d72e046b6bd4",
            "16aeb0e130fe40cdb85d07b475be7065",
            "89c9a9d4b5dd4ca38ab9c56ac05b851c",
            "bcd1ee570ae04f23a6a7d330490ceeff",
            "9dfa528136a44470a4d6265b8400b72f",
            "b8c09bdc4ea34d5094dfda9f27c6d69a",
            "c2814c5274d142f08000b805904c6cb2",
            "af44ea3b86594c2b9645a1507abd02e3",
            "4b64392ac48f40f9b99e8b4537c7ff16",
            "82094929eff84d92965c9204aae7a56f",
            "3636146ec0c44ed6b1b62b32e3ebc1ef",
            "2bd3f480cc7349cea11858f34ea21675",
            "d85e97e525ed478bb0b84ed061f2e16e",
            "32cad59617724dc1b4c6122bb83fad9a",
            "d8af45522e2143d39cb800ebebbf2428",
            "845bea1ae837482cab41913de153a40d",
            "d7735bd13b4f4b889db294fc7223ef6b",
            "c5493a4b72a845349c55c0cfbb08e6d0",
            "5a9010d5e12e416bbab2a8ded8178a91",
            "c881530aae434d2b841b8b5b234437b6",
            "a87104b984e64895b21e28341b2c3534",
            "6b3d5d016b914408b6a7a3370922ac89",
            "43bacc87287b49058d94638039019a63",
            "06595f5db6c84829981c417e57fcf7bb",
            "77a4df986b1c4ea380f0547192454401",
            "16f33b0870cb4e5181a3fe6675c133a0",
            "83ef198a83f74cafbc16994d1e9a1c05",
            "ae539a4968a3460c97135d3d7d301a3f",
            "57014339c3714a58a259f0d18b6f59b5",
            "574997adc5ed4d2993d7c1a14b695d3b",
            "6788d6f01a1743aea9664253a7af9b39",
            "85ac829588a94674ba6b0809385dea7d",
            "4bdd1f52606f463c9fd9e1eaf64fc36b",
            "d7b64268f5334ad5abdafa59015f8926",
            "62477977c74e4008b07a2ccb05e2038a",
            "41c4b30aa8b54d169d70959bca31ec53",
            "b6f8d541e6d74b1ea7201c17012205ee",
            "0720fdb2ce6a4fe29b926f1db51f3496",
            "97abf81adaeb415992c6908d7fdc3722",
            "431f6d9310b7496d8af06851f3847759",
            "e938d9d158e94c1fb9733b994b79efff",
            "8d0a0f3875c6402ba60bac8ed34efc8f",
            "41e436b95ca04035be8e63e2d26574c2",
            "f6fb602fb9144735a6d3d76d9b46eda6",
            "9386fcf485e541e5ad445b4dc8b7baa5",
            "ca2be5aef85e4586ae42f56da91ff071",
            "6a1f16cbef39467a9464760db7267b28",
            "beb33f900f72403b91fa3a15733112ec",
            "ca16aa8fc1aa4f059f9db65f6d9462ee",
            "c6d73576e0134ea4b357d1b1ca1e3035",
            "ddb4c88579bd4ece9c14972a9dd2fad5",
            "4f1b7752317241bda38a421be12e6f80",
            "ff7fadaa45544254a2b45c7807291c65",
            "76ca7ca594e64523bd4c9d1de8d9f3e9",
            "cfb54ea1e3eb4ebcbea05f900110a14d",
            "9554c3f20d4e45059b8e6292632ada24",
            "d4cb9b1b65cf4fc98b729cfdcbd2c489",
            "86366d18c58d4465b1a5d5ead41a434e",
            "113b92b060014e27ba14837eb3625583",
            "b6354156b4e84d1ca00652a7fbdf2e01",
            "88887bce8e2448ea9c89bbd436390778",
            "b294449c0f254b9b83d17671ecf88df3",
            "6f9d8687051c4a0faf9167cca6f71916",
            "86cdcca3619e42b892c23050a0b1210d",
            "1e6dd8e134e04b0aa1f96b40a9f590c1",
            "f4a6b25e75c041968692358e72f733c1",
            "2cb100eced394578bb89e61093dbb56f",
            "97a471a4f99c4dd9990a0b797be84bc0",
            "00073f159b7e4375addb370ec9503a41",
            "bbb0260ebad648cfafaf897ac49b5238",
            "ae1d922026e24d87b612f77a6ab79c9f",
            "d537af6594b64c03a1b60a8e1133e23d",
            "c5c81d33df064bebb11c5bf0f87a25e4",
            "82e9a03bda2d4750a9b7f1ca16f1ce44",
            "810a2cfa6e68405b9ef4d789ffbb5f39",
            "dcb20d997f1f4106a39407c0262554ac",
            "19c2e2b53bd14728943b0a79f53ca7ba",
            "188df7c8a3704727a7b9060656c52861",
            "16c359536713446db8806380c413fe0a",
            "633323e845dd41438ec3af50c30abb4f",
            "2b385fd6ce3c4dd390c9efeb01eeea5d",
            "2f562e38ee214609b33c1582adac07e7",
            "6eb1f0cd05794ccfabb0826d949995c5",
            "de2a50a480dd4706af618d60d0e7db80",
            "f6b412def4c8469599d9b49d4dc72ac3",
            "937c23149ec9402f8940f3b70870920f",
            "aab2506fe22f44bfa6debdb63d568910",
            "5f9bfda8bd6d47aa94c55f4cfee31519",
            "11a6f4d4dc6c42b390d0703f6b722b1b",
            "8d0c59e569164851a558af0a3b91afed",
            "6f6d617d35e54407875eadc4849aeea5",
            "20078382efb14d9ea4b55fb8a40b151c",
            "b01601a0ffb844a4acd5a18e0f8a9d64",
            "10a682c72bb64c5ea0ee8ac3b7af3880",
            "96a26fb14a3d4de395a37b2a1157d8b3",
            "bb21650c39254c1fb5130a2a6b0911f6",
            "b75ec3a8369348bb8c2bc579bce8af9d",
            "b7c9d1b307b74af19a6644e56b154e9a",
            "85944c112a19478da9c6e73fe1439996",
            "f70decbabec94008bc8be24983df879e",
            "5316d200bc2748faadf46b900da08d82",
            "9e7e3499af9e4d4d9873af0bbe86de37",
            "bedcc36b0a7640b4b30ec5d618bd85d1",
            "100b6643f4f34dd88a3a3caad4934f73",
            "16a47bc0ec8e4459830f2d78263b64b0",
            "924e90ca1a6e4da4886314d9d8a71ad1",
            "716612c15c6d431c92cb86b52fd23453",
            "0b7b7c4b74594e04b8b43c567c443859",
            "67ac68a6a25a465ebd3be26ce34c50cf",
            "9293b44a9a2c42b6bc99b99e7360a230",
            "cab15d7b66a84cd38abf0a2ef7b837dc",
            "3f69853a97374b35bf2e3858c915c3b8",
            "23a9b478acd44fbd889c4d7999751c0f",
            "b749cc5317f54381b9ec8ca757fbfd0b",
            "9092867168354f0fbc4d7526b393e080",
            "625f9f64009d4ec6b9aa7b872641b54c",
            "21a87000ff0a41df89c13f624c2c20d5",
            "771dcd121b6d49c1830ef432b4e3dee2",
            "2b1ea240e5334919a27c2a4fae47559c",
            "a8ef656525924940a88ba6688468caa7",
            "7494b9356229460db70dcdf80d82ecc0",
            "032c223af5664afc8820f13b33725843",
            "3aaecc4a09c54fa0b008bd21f4be840d",
            "68574004aca64e6cbfe32784ac08658d",
            "15c10640d90645f293ec947af17c7879",
            "3a199b740a6c4ec4804dd1c6abe9b79f",
            "dede39113f7e49c4858227ee88c88f89",
            "d287842f51c145158a5a8986aae88031",
            "39cb72fcb6be45b297a808da85f234a0",
            "ed365a2e598c4de2b0fd3259ce671101",
            "55a173b9fcc24322a4b2ce13619f685a",
            "e0878781c7b8493ab25445627709bf5f",
            "0fc6d82ecee948d8a46ec5d1fa2679ad",
            "b85a96c7c8894e17add1898ea9f4ef02",
            "541c47357be44829a0204c5275f147d0",
            "75724db3983e4194b4ef023c0a20361b",
            "b1a9a4df4baa43dfa647b44d62b602ca",
            "907a81d0b8a948a2a91bfcc63a157daf",
            "c9abe515848d441f92a2e927c67ddea4",
            "d432d5cd0d8a4d84a9bf9b412058fb50",
            "2e5669550f5e45af9985dbea858ffe12",
            "37d7bbcceebd4c30be5975ad925c1012",
            "a12cbbb658c54667ac03c08c77588bb1",
            "ba92728f6e374c9c93c7947a7568eacb",
            "df64ef963eea423c83d6a400ac3304e8",
            "7010f2785e2846efb1965aff28c672eb",
            "dff0d17434224b5aaecad18f0785e4f2",
            "4f8a2ec038d14d7c96be1286e00884ab",
            "325bb3604c2745469a0f74c08ca0f5a7",
            "82b4cbd8ca474ab08d3a3950afce4c61",
            "678181ed5f8c40eea63d37e96413272f",
            "99e5523babe84570ac7274328df6e55b",
            "a8795ae305b3478fa70cf4ba551771fc",
            "194dde99d167494b95db0c5364a8849f",
            "b5f58440390a4c148a38a10a13b66ae1",
            "0ebad828fc0341c3adb05f06d57ab6ba",
            "8b6e2db9dd484e4e87a25cf5b346ac24",
            "488f81747b97427b9ce7c2c2a6422783",
            "a9e6d2bcd72f4ebd939613f0efb54ee0",
            "69a16531fc4c43f2ab42767fb48db7dd",
            "3aa86a6045ca4068bcfa91c889aeb73a",
            "d492f4d167684984bc81a37714675dd3",
            "ba5555e68ba742a580798662f1593544",
            "61954000e65d45fd9045e4b1b933d624",
            "e961d68a700d47ab9c84f63b5ddc6198",
            "1b40a2d125fe45f9b1730133ca18408a",
            "b134e83357cf4191927065f950c6fdeb",
            "e2662e636ffb41cabef0bf9f583ae13d",
            "11c8906d575e4b6ca5481f43995da6af",
            "38cb71d4c9644946966b9696b4b332c1",
            "caae684f2c334ded85a0c97b4560c36c",
            "3e541d4372734057a2cd15328ad0273e",
            "752d8e368da4445ebd7abd220313bc5b",
            "7e2c7d1234b048439a18bc60e0a86ccd",
            "562241ecdf0f472098c30bf7ad417fe7",
            "38b6cd024336478fb5a33d388ab689dd",
            "3e4d99ac3b7b4d0db3a12e1e3748f053",
            "e52589b401c244e5b9ab59c71b3fe8a6",
            "c16742d341a84f3f8b77759238cae471",
            "7a3eb3e711b74ff5bf4a2772ee701d27",
            "f0e53b90e6024032b4ee5314fa32395d",
            "129c7e6a61354f5a92c7d78f38d91a73",
            "ee76badbc9e74e12be0db51e5339a938",
            "e752144193bb4c7ba2fd4d64cc11b625",
            "d6d573f204874106988b4099726352ee",
            "793eb1209b984d39be787a3f1b29bc93",
            "e8c2ad59f1b54b57a97b8b0487a53f39",
            "8cbfc946dab44b07be54ce15b78ff8f7",
            "fd62b2f46e304ccca9befdf0e61d7b0a",
            "e18d9ad072b7433295830d6b74453627",
            "d71a36cb16e84db4951fbf27e50a74aa",
            "8c66d0c5deda4aca91e21e41eeaa6b60",
            "c56f8250eb194ef89edd29d62a0ccb32",
            "a5c58805fd574313930201e35f62eec3",
            "1d7c3e820b8e489d92851c6bcb8c51e0",
            "bb8dbd0a1a524b9bb6fc442ad469579f",
            "a192c2f7ad2f41359b4bf8e106d4c3f1",
            "d147c646bc264669ade4f41232e9f276",
            "cc01e4f7ecff41a184184ed6523ecba8",
            "31f995b785c9487a8593cc666ae6dc11",
            "5eeaee217899423884d8b17aa1da44e1",
            "d53b0729323044988ab3a9bac2d4e701",
            "819de7a0797443b88a00d7b95d0ff92a",
            "65df5040da69435ba8c20cc8c5e748c8",
            "38fdef8bccb74d439e48c5f679e46119",
            "15599c748fb040549cb730e71a962dcc",
            "cca580d8f840464db47cf2483be5b304",
            "813134bc0eb64a05909399104d58be24",
            "6d639facd165459b992bb9aa6860d7ef",
            "29d4037ad7594c099f3803870cff0393",
            "f4a47f8360074f0a9044b2af3e4d7909",
            "4dd08c8557d047e88e47a9b167ebf612",
            "5ae6d2672c5049fe8c3662f8b18805ff",
            "4d06807549354a8bbce3e29b8914c15a",
            "f30e26a312d9440fbeb758d530f626f1",
            "5e8f624960b740bbb2f7b6e6c2fab9a3",
            "06de3068cbf24862a9b0fca2de4d9af4",
            "bd28fde6f6fb4cbe9cb189070d219d98",
            "66ce747d8d014fad97e653eb490ce954",
            "bec370c73f764f95a7a113ad4da26557",
            "143e19329b2f469a88e24b73f530e030",
            "56dfeddab5244ec6a8807a896d1f43a5",
            "053996befd164da9b1652c7d0e3cafda",
            "965c46e055584a82b29c616f33ebccf9",
            "e748f20e023041f3a0bb3b38914bf24a",
            "fa546018c88a4b9ea02012654895a746",
            "94f18e78e2d742b6a3c2fd27fb938259",
            "52cea86d42b64e91864018672008beb4",
            "6635ea1a328f495aaa815d9a2a47cf1c",
            "12e309dd99664249922802bccab77da6",
            "9598e910700d48429683ef815d595155",
            "e84bb8bc3df7441ca1e7284f0c976842",
            "edd8d6181d61435b9ca85800389a64af",
            "f56a3b7b96e247a8b30937773c8c6d03",
            "794ab36576cf4c2980a7ecd57ca18d2f",
            "cf55acb845f540cf87fc0be37b864bee",
            "05dd8c0aee9a453f9622c4cdec1069ab",
            "18cc676ba4f648819a7768d673c669f9",
            "85240e7b8b56470b94f034a9f12f7850",
            "1c9a18b5b564483389ad0a3cee8aaf30",
            "dfeb8fadffb44ab4b94eca442481ea5c",
            "738a942c566c40638b12d8de5c4ab997",
            "6a62028c9b124c9da0fd3e2a5c7f80d7",
            "5bc2c5b527a64a36abba837721e5f443",
            "8b9d04ef324241a39a21bb883efdb88a",
            "2edcb541f1ee435e9ba638ba178e7cca",
            "3cb07cfb9d8a406e8057c62caf3f0219",
            "b7ed6275c7404615ba6528b2bc393ae0",
            "dea52e0581f54f5f949a95be6d975cf2",
            "f2cbc31443aa47adb2e74c4345bca18e",
            "292f91bd24cb44598fa47b4a3dd09119",
            "915f8190942f4abaa7b4bcb02a584454",
            "f63a1ac3b0934f519b795f5a2f560a48",
            "35f4c4d38ec84a60b6352aa99adfd89c",
            "446ccfa1277645caa6f70e47800c9e43",
            "8c030566f8a44d07a682db5c8ea58955",
            "6509663888774b8fbf3cde536a423c3e",
            "e39a78f5cace44bcb762a3b32049010d",
            "5fd54e07e2744882807ed3ecf857e544",
            "3cda324cbe904dc8b44d12a9099a5050",
            "2ead824d3083472c80cfb786cb4f9152",
            "93134c44586c4f67b0ae7fd4ac0eba57",
            "e8fa8437942d4281b6be0be9bcb603fc",
            "8314d74f9d4d423cb4f1432d66d9af74",
            "fdf520395ee04362ada0bc8af0747446",
            "1da3f249ad7c4776932e29400b3811d2",
            "40002d05dda94fe3b8357359b91bd220",
            "20447be793d84d8cac640b5b318941f9",
            "721f6ae801f848d3b50e8abad5e7ac36",
            "5d0eb472e8ea4aeb916c2df761e1b178",
            "7dd937e8ee034b1a96605b9b421d6233",
            "3204b85c7a274f4d8400ff4cf99abf67",
            "8407c09f9fb94e30bf67fabfbbb71af1",
            "4b65a861a728450489c172bcd4f3e680",
            "3696a2ec51fd44b4b06ac9cf6abf9dc8",
            "9657008a39b64ba0af183af35695a0e8",
            "848ea07702b446759d82c9c63087b70b",
            "fdef95ddb8574fffb95809b87b776e51",
            "527b51380a2d48cc86cbd50931575b95",
            "e89e732a046b4381b631da47d49c8dc3",
            "cddd3eb36e7146d197965a50d42ae344",
            "930dd91fdf8e4f61a7a538623cfb1762",
            "137dd396814f4018913db9d1b41ebc3d",
            "3c6f863a92e34b2a9c6bd51b5eaa51a1",
            "08d57e987b9940e78cb8e5d0d285c165",
            "f09e513e5e41479c9a8f211945da0b18",
            "d8835b967eab489381c661f1ababcb66",
            "66acf5c9b01c41148e30640e06c90d26",
            "b8ced13375e742fb8ed1fe06e07adff3",
            "5de9baf9eb12461f9594db65ed3bdbf7",
            "801c33816556437b838d8c35643b1c64",
            "6aeee743f1494b29b45a92728701ed04",
            "b9869bab632140438265580fe4c07943",
            "469efd3cd8514eaba391dd69e455d611",
            "0e5d1e5c62b84962ba2ce09ea6d1bfc8",
            "7cb61ffcd472416887046382c683cbe2",
            "983ca17fd9074043b5eda8d0324208f3",
            "5369f9c47210459ba4da9c48a0ba8155",
            "0f053455c34a4ddcb781a4af1eef9c0e",
            "b02fb5b9ae834ae0a25d7f14c945fb86",
            "ffe811faf8e84d3bb0008d32085ce2e0",
            "32bf12e6fea14b9fac45e12a47fb304c",
            "bdac37518a834f06916a77e978e0a94f",
            "f510fad6c4944afe92c3958113a51584",
            "50c6ec5c82534c428be1fdc9f7a6c236",
            "52f27fd9665e45d08580061e820bc966",
            "910ed60756824e119d6881fdf14c0229",
            "9a724a1d441142b8b9d781d59fd42fa3",
            "55bf2b0180084ca4903fbfd7dddf5b93",
            "e238771fe9554cf39fab6d48dc434506",
            "905b2b1878f6466486c5667be993784a",
            "40846c703cd34cb6831083dc93bd2108",
            "ed3fd3e8f89e42978cdee77322575d39",
            "57508509c2a24a12beeedc88712ef957",
            "744aa61ca0f34d0587241ab291e8a6cc",
            "d190df2951f1490cbf75594585c6be63",
            "67c9b214a23c4477afb3b5e80ed593d0",
            "e43f2f78a4794f7b8dd511d335c36ce6",
            "701bcc0d6f86443ebb1ba5edd10d7e7c",
            "995bd2b1ec884639abd1184b9f6ee023",
            "a8672d246865460eac43ff3b504a0d75",
            "92706596069c4bd2a80331057dffcac9",
            "d464a38ed15041f195ab6299780255a8",
            "bafdbc3df70a469c8b59158e69cbf66e",
            "bb65d6c9bc6c486aa1dd24a364bd0421",
            "3e8f9222c16e489db3a4017226ce94a0",
            "fcd0bc6ef9f94f6dabecbbff33bdacfa",
            "458682ce398945df963a1645a4f801d5",
            "b7e997fb0ae94b6294b5fd02a3f62a6f",
            "bd6bc75cf682494a95799c0d5da48865",
            "3b6a7eedef9f4692bf506e3e3dc87eb3",
            "f46a81fe38814412946d54f649cc8c90",
            "f68ddc1853b5404e988cd3d33d1891ad",
            "10536a3cd98a482d89046cbda1f01db5",
            "b7bfa8c8aaec42a499bbf48d299ad9de",
            "3e2cb1ad40824154b0007c7934569f55",
            "d68f196887ec4e3fbad1eb3b62030617",
            "c8268557a7c048b998b83280c00e3764",
            "e47364b656d64db09b7f32fb042c142e",
            "f7c7f6dca238412898b6b8d8ed4d9b9f",
            "4877b4bd13a943cfa3f47a99ab324fb1",
            "db371bd861214313b77fa769bd047f6a",
            "10df45591374434589904fd123ec059e",
            "4d8904e56e514e82aec01646bffdcebd",
            "a36ebb54173245a7958028e61ca28152",
            "0f2186e33d5c4959b0b4d8619325502f",
            "d4a2962b07d140168ced4782e0e6a3ed",
            "ca1145b5312846519a7401f51052274b",
            "c94e2f8e638b4241854e82d19a7a51db",
            "d54c72bc7c01408980dd2149bc9c1d48",
            "2781a50ce0bd4e3fae8856dd5a6e928c",
            "e0997af6dfc14c0f9a13bf277e1aa915",
            "0bc544f437994900a9f9fcc82d41df69",
            "7de3ad4c157b41e7bf8f3a9681a9aef4",
            "11841fc752d84106a15c754f4ef342e4",
            "9c1fd904008c480c87eb03a586c29bc2",
            "217853d63f214a0fa9b50e3cd6bdc7da",
            "9e6369e4cee24f09a6bb942ba4fbe332",
            "bb77626d0fee427881012af8bfac91b4",
            "691b12a6249d4743afc3c410fc15e6ce",
            "68a8312e677843dd8e3d457b3251839a",
            "ce6f99a9c376445aaf9d8d8ca2b3a0a5",
            "373b303fd2f84877af318c06874277a0",
            "a0211a22604248319676f38dc22a78a8",
            "2d7d13000f02432f906b7328db834686",
            "655e668faf4f4ca6acb15abbd84d9151",
            "0c98b7944ea34be39a41219e368952ac",
            "e7fa47555b8847d48b43e70c1742283d",
            "bc33f0f7ce77479ea9ffd1fbd2f52596",
            "acdb71729d7e40769a1aba987c122402",
            "84569cd665b8499aa05c593d4a20fa77",
            "d6f284b48d1a485fb29f8b0081076c7b",
            "03a4dce223334fb88da4cfe1bf9a4c5d",
            "cfd02bdb3ff54a119169b5c316cb8671",
            "03c57edc05f540a190b0c8d47ad91e96",
            "c07d3d89d2d941659da4bef31762031a",
            "8c5727a32ff04bab9205ca7a73d20b46",
            "231c096ffc4248348d025d456197ce60",
            "facb47478c44408292911eaa79488a98",
            "20797fca24d54f9e8fda279baa80ddf6",
            "bf621b08cd444c6e9097a6db12b79673",
            "da3fc063d322459d8596107a6ef57c81",
            "0079448623d7460898eb467523c788df",
            "cf86749023cd4bdaafb3cd107d6649bd",
            "2edc4f63e8a14638958d8c9120070b42",
            "44940d6ccfc04b23b7b5b2fed9bfbc9a",
            "74a425d701c64741bab5ed9d83fa4268",
            "18272d2a0d7f4471aff9df12797a5fae",
            "f1c5d2eb07e341449a32bfe5872c5e0d",
            "6e745b680dda4f58a9cfb02f75a333a5",
            "9cfece3e5b5a4136bcb8595da192cae1",
            "e85405a881a94dd78b0b64dcae7d73cb",
            "3458f2c50bb24636bd43b6cc931220ff",
            "388606f723ce4e83bab06dfffdee3375",
            "2ea3156d82b5489a9c6c1fd7e8e05e3c",
            "a38bd76094564130b2d09ff8cc8ec220",
            "0dd333c4bf3c4bc9a3e99e6c062da52c",
            "ebcaceef655841c2aac8836ab8e01296",
            "21569e699f8445abaf968f25e4b8b67d",
            "0484ebd89b2444f1b38cfc78c1c960c0",
            "c66c1f7a2bca4509b54b229674bac51c",
            "09d5e48b4fba46f1ae08c47587948128",
            "2f43de5591174987ae60500784c9c004",
            "762695fe4746402d8dcbbbb0bab9c42a",
            "fe02600d4e994f8b8034d7df3ff5d45b",
            "4c543cc65528407ca2660b61aedab21d",
            "1807e4f5623949e0b3c10a8e8264972d",
            "1f210816a3fe4f1dbb6e57fc3866aff2",
            "c6bb0573a5bd449ab19edcd6f5c8ad5a",
            "28ef331c801d496082b1cb42c7d27b23",
            "ed5903b67abe49dfabb825ed004ce391",
            "982b55b923244a728248efecaec4e704",
            "d990fbaec0314440b643866084ae9c5d",
            "3d65db37c745427ab08e51232e5ab0b3",
            "973dc1017532461d9f714c6ba539fedc",
            "968bee37914b4637b3afea2d39fb5d30",
            "dd8dfefd74ce4161a45415735d80020a",
            "d06ca340544b45bd803d14f5608edfc4",
            "86e5b0a2022d47f689b65e6a72891729",
            "d2257c0520d1499898c304ac5dac213a",
            "2dd6beac35bb4e48ab9f718bf65834c2",
            "5f499f87dd5a4ce99022831439ad117c",
            "cf9bcdc00c7842d4be89055415eedb4a",
            "177eee6c39e0413a829a99f34f0d318b",
            "927a6da54d544e7f80c791ff23e1963e",
            "b3b6d107ff764c16ba7a10b657ec2d19",
            "0e9601a9899a457ca21248c5ec288918",
            "20ffd436d8694784a86a1e48f4e08eb2",
            "74cb7737875e40e29439aa8907b7b2e7",
            "6b83c488b3dc4a53907c3c74f0afc3a8",
            "1aebd94b2f414220b775910063d8a1e0",
            "b20cc89bc3ed44e989c1b288133e156d",
            "3f3af9974e8140ff977cedf9ab07b3c2",
            "43a026ea252f407b9fc1e06753e60dcd",
            "dc4d5b8cd31e4bb394d4db9c68cabcb5",
            "db4e2028ea3d4a1e9af1bbb3c5d87168",
            "9f57e4359a084f8c88844129f7bbc1b4",
            "ca119e87ad634cb39838d69d2e9c2acf",
            "3e71dda793384021a0dd16fcf1872704",
            "6c49572bd81b405abf47b64af7e5bbc3",
            "b0e9d2488dfa48a69aeb8cfa33002479",
            "556a213f31a84dfea0d418e2317c4cac",
            "f65c0eabb4ea4afd912f5d07d59ea735",
            "c5894d1718d74d66a39e4e0c4750ceb8",
            "bee6f62656df4f258886f29d5d9155f3",
            "afb7c009c2884685985f3bb14f12e58e",
            "5ed9cdda8e9b453db0d4d88fbf9a0960",
            "55a3e0378bc64aee9022510a34415912",
            "d5e5864038e94eca81514ebd1a42a09c",
            "ae7e2087e5104fa587a32857ca2a6b2e",
            "e434678700f943c58f9af8c8e7886873"
          ]
        },
        "outputId": "b6b9cc9f-c5c2-4280-a92f-a47422e25888"
      },
      "source": [
        "epoch_itr = load_data_iterator(task, \"train\", config.start_epoch, config.max_tokens, config.num_workers)\n",
        "try_load_checkpoint(model, optimizer, name=config.resume)\n",
        "while epoch_itr.next_epoch_idx <= config.max_epoch:\n",
        "    # train for one epoch\n",
        "    train_one_epoch(epoch_itr, model, task, criterion, optimizer, config.accum_steps)\n",
        "    stats = validate_and_save(model, task, criterion, optimizer, epoch=epoch_itr.epoch)\n",
        "    logger.info(\"end of epoch {}\".format(epoch_itr.epoch))    \n",
        "    epoch_itr = load_data_iterator(task, \"train\", epoch_itr.next_epoch_idx, config.max_tokens, config.num_workers)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 08:33:20 | WARNING | fairseq.tasks.fairseq_task | 1 samples have invalid sizes and will be skipped, max_positions=(1024, 1024), first few sample ids=[326674]\n",
            "2021-08-02 08:33:20 | INFO | hw5.seq2seq | no checkpoints found at checkpoints/rnn/checkpoint_last.pt!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bf7ce63a840349ae9bb72dced9b02515",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 1', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:41: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:35:15 | INFO | hw5.seq2seq | training loss: 7.0785\n",
            "2021-08-02 08:35:15 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d58e36f69fb849109fb31716510ae80a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /pytorch/aten/src/ATen/native/BinaryOps.cpp:467.)\n",
            "  return torch.floor_divide(self, other)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:35:36 | INFO | hw5.seq2seq | example source: but the point is , it doesn't feel that way .\n",
            "2021-08-02 08:35:36 | INFO | hw5.seq2seq | example hypothesis: 我 , 一年 , 會 , 很人 。 」\n",
            "2021-08-02 08:35:36 | INFO | hw5.seq2seq | example reference: 但重點是 , 感覺起來卻不是那麼回事呀\n",
            "2021-08-02 08:35:36 | INFO | hw5.seq2seq | validation loss:\t6.5587\n",
            "2021-08-02 08:35:36 | INFO | hw5.seq2seq | BLEU = 0.07 13.0/0.4/0.0/0.0 (BP = 0.683 ratio = 0.724 hyp_len = 80915 ref_len = 111811)\n",
            "2021-08-02 08:35:37 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint1.pt\n",
            "2021-08-02 08:35:37 | INFO | hw5.seq2seq | end of epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fa08453f6ff34f0187434138e43c04b2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 2', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:37:37 | INFO | hw5.seq2seq | training loss: 6.3543\n",
            "2021-08-02 08:37:37 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c19f12cf4b4f41a3a0d32dfb777fab3c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:37:55 | INFO | hw5.seq2seq | example source: we can push through that .\n",
            "2021-08-02 08:37:55 | INFO | hw5.seq2seq | example hypothesis: 很簡單的 , 很棒 。\n",
            "2021-08-02 08:37:55 | INFO | hw5.seq2seq | example reference: 我們可以衝破這些限制 。\n",
            "2021-08-02 08:37:55 | INFO | hw5.seq2seq | validation loss:\t6.0070\n",
            "2021-08-02 08:37:55 | INFO | hw5.seq2seq | BLEU = 0.76 23.6/2.8/0.6/0.1 (BP = 0.572 ratio = 0.641 hyp_len = 71722 ref_len = 111811)\n",
            "2021-08-02 08:37:55 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint2.pt\n",
            "2021-08-02 08:37:56 | INFO | hw5.seq2seq | end of epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bcd1ee570ae04f23a6a7d330490ceeff",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 3', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:39:56 | INFO | hw5.seq2seq | training loss: 5.8180\n",
            "2021-08-02 08:39:56 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2bd3f480cc7349cea11858f34ea21675",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:40:16 | INFO | hw5.seq2seq | example source: thank you .\n",
            "2021-08-02 08:40:16 | INFO | hw5.seq2seq | example hypothesis: 謝謝 。\n",
            "2021-08-02 08:40:16 | INFO | hw5.seq2seq | example reference: 謝謝 。\n",
            "2021-08-02 08:40:16 | INFO | hw5.seq2seq | validation loss:\t5.4212\n",
            "2021-08-02 08:40:16 | INFO | hw5.seq2seq | BLEU = 2.08 22.7/4.5/1.2/0.3 (BP = 0.823 ratio = 0.837 hyp_len = 93608 ref_len = 111811)\n",
            "2021-08-02 08:40:16 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint3.pt\n",
            "2021-08-02 08:40:16 | INFO | hw5.seq2seq | end of epoch 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c881530aae434d2b841b8b5b234437b6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 4', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:42:17 | INFO | hw5.seq2seq | training loss: 5.3769\n",
            "2021-08-02 08:42:17 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ae539a4968a3460c97135d3d7d301a3f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:42:41 | INFO | hw5.seq2seq | example source: those men would eventually be convicted of placing a van filled with 1 , 500 pounds of explosives into the sublevel parking lot of the world trade center's north tower , causing an explosion that killed six people and injured over 1 , 000 others .\n",
            "2021-08-02 08:42:41 | INFO | hw5.seq2seq | example hypothesis: 他們的確認 , 一旦有一千五千名人工智慧 , 大型的公共空間 , 大眾的公共空間 , 大眾的平均 , 一千五千公里的土地 , 大陸的土地 , 大陸的地面 , 大陸的土地 , 大陸的地面 , 大陸的土地 。\n",
            "2021-08-02 08:42:42 | INFO | hw5.seq2seq | example reference: 這群男人最終被指控將滿載1500磅重的炸彈的廂式貨車停在世界貿易中心北塔的地下停車場 , 爆炸造成6人死亡 , 同時致使超過1000人受傷 。\n",
            "2021-08-02 08:42:42 | INFO | hw5.seq2seq | validation loss:\t5.0637\n",
            "2021-08-02 08:42:42 | INFO | hw5.seq2seq | BLEU = 3.55 22.5/5.9/1.9/0.6 (BP = 1.000 ratio = 1.179 hyp_len = 131843 ref_len = 111811)\n",
            "2021-08-02 08:42:42 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint4.pt\n",
            "2021-08-02 08:42:42 | INFO | hw5.seq2seq | end of epoch 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "41c4b30aa8b54d169d70959bca31ec53",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 5', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:44:43 | INFO | hw5.seq2seq | training loss: 5.0310\n",
            "2021-08-02 08:44:43 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f6fb602fb9144735a6d3d76d9b46eda6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:45:03 | INFO | hw5.seq2seq | example source: but this is really how i go about creating these photographs .\n",
            "2021-08-02 08:45:03 | INFO | hw5.seq2seq | example hypothesis: 但這真的是我要如何創造這些照片 。\n",
            "2021-08-02 08:45:03 | INFO | hw5.seq2seq | example reference: 但是我真的是這樣做出這些相片的 。\n",
            "2021-08-02 08:45:03 | INFO | hw5.seq2seq | validation loss:\t4.6510\n",
            "2021-08-02 08:45:03 | INFO | hw5.seq2seq | BLEU = 9.34 34.9/14.0/6.6/3.3 (BP = 0.922 ratio = 0.925 hyp_len = 103459 ref_len = 111811)\n",
            "2021-08-02 08:45:03 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint5.pt\n",
            "2021-08-02 08:45:03 | INFO | hw5.seq2seq | end of epoch 5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4f1b7752317241bda38a421be12e6f80",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 6', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:47:04 | INFO | hw5.seq2seq | training loss: 4.7607\n",
            "2021-08-02 08:47:04 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b6354156b4e84d1ca00652a7fbdf2e01",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:47:27 | INFO | hw5.seq2seq | example source: so that is a fact .\n",
            "2021-08-02 08:47:27 | INFO | hw5.seq2seq | example hypothesis: 所以這是真的 。\n",
            "2021-08-02 08:47:27 | INFO | hw5.seq2seq | example reference: 這就是事實\n",
            "2021-08-02 08:47:27 | INFO | hw5.seq2seq | validation loss:\t4.4167\n",
            "2021-08-02 08:47:27 | INFO | hw5.seq2seq | BLEU = 10.49 33.1/14.3/7.0/3.6 (BP = 1.000 ratio = 1.100 hyp_len = 122943 ref_len = 111811)\n",
            "2021-08-02 08:47:27 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint6.pt\n",
            "2021-08-02 08:47:27 | INFO | hw5.seq2seq | end of epoch 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "97a471a4f99c4dd9990a0b797be84bc0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 7', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:49:28 | INFO | hw5.seq2seq | training loss: 4.5860\n",
            "2021-08-02 08:49:28 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dcb20d997f1f4106a39407c0262554ac",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:49:51 | INFO | hw5.seq2seq | example source: thank you so much everyone .\n",
            "2021-08-02 08:49:51 | INFO | hw5.seq2seq | example hypothesis: 非常感謝各位 。\n",
            "2021-08-02 08:49:51 | INFO | hw5.seq2seq | example reference: 謝謝大家 !\n",
            "2021-08-02 08:49:51 | INFO | hw5.seq2seq | validation loss:\t4.3084\n",
            "2021-08-02 08:49:51 | INFO | hw5.seq2seq | BLEU = 11.16 33.7/15.1/7.6/4.0 (BP = 1.000 ratio = 1.141 hyp_len = 127591 ref_len = 111811)\n",
            "2021-08-02 08:49:51 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint7.pt\n",
            "2021-08-02 08:49:51 | INFO | hw5.seq2seq | end of epoch 7\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "de2a50a480dd4706af618d60d0e7db80",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 8', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:51:52 | INFO | hw5.seq2seq | training loss: 4.4722\n",
            "2021-08-02 08:51:52 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20078382efb14d9ea4b55fb8a40b151c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:52:13 | INFO | hw5.seq2seq | example source: these differences became a source of inspired artistic celebration .\n",
            "2021-08-02 08:52:13 | INFO | hw5.seq2seq | example hypothesis: 這些差異變成了藝術慶祝的機會 。\n",
            "2021-08-02 08:52:13 | INFO | hw5.seq2seq | example reference: 這些差異變成了一個源頭 , 富有靈感之藝術讚頌的源頭 。\n",
            "2021-08-02 08:52:13 | INFO | hw5.seq2seq | validation loss:\t4.2015\n",
            "2021-08-02 08:52:13 | INFO | hw5.seq2seq | BLEU = 13.34 39.7/18.6/9.5/5.1 (BP = 0.972 ratio = 0.972 hyp_len = 108716 ref_len = 111811)\n",
            "2021-08-02 08:52:13 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint8.pt\n",
            "2021-08-02 08:52:13 | INFO | hw5.seq2seq | end of epoch 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f70decbabec94008bc8be24983df879e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 9', max=791.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:54:14 | INFO | hw5.seq2seq | training loss: 4.3883\n",
            "2021-08-02 08:54:14 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0b7b7c4b74594e04b8b43c567c443859",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:54:36 | INFO | hw5.seq2seq | example source: it's not .\n",
            "2021-08-02 08:54:36 | INFO | hw5.seq2seq | example hypothesis: 不是 。\n",
            "2021-08-02 08:54:36 | INFO | hw5.seq2seq | example reference: 不是\n",
            "2021-08-02 08:54:36 | INFO | hw5.seq2seq | validation loss:\t4.1456\n",
            "2021-08-02 08:54:36 | INFO | hw5.seq2seq | BLEU = 14.20 40.5/19.1/9.9/5.3 (BP = 1.000 ratio = 1.017 hyp_len = 113737 ref_len = 111811)\n",
            "2021-08-02 08:54:36 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint9.pt\n",
            "2021-08-02 08:54:36 | INFO | hw5.seq2seq | end of epoch 9\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "625f9f64009d4ec6b9aa7b872641b54c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 10', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:56:38 | INFO | hw5.seq2seq | training loss: 4.3254\n",
            "2021-08-02 08:56:38 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "68574004aca64e6cbfe32784ac08658d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:56:59 | INFO | hw5.seq2seq | example source: so think about when you have friends , families and coworkers in california , on the west coast or in other parts of the world .\n",
            "2021-08-02 08:56:59 | INFO | hw5.seq2seq | example hypothesis: 所以 , 想想看你有朋友、家庭和同事在加州 , 在加州西岸或其他地方 。\n",
            "2021-08-02 08:56:59 | INFO | hw5.seq2seq | example reference: 所以 , 想像當你嘗試聯繫在加州 , 在西海岸或者在世界的另一面\n",
            "2021-08-02 08:56:59 | INFO | hw5.seq2seq | validation loss:\t4.0919\n",
            "2021-08-02 08:56:59 | INFO | hw5.seq2seq | BLEU = 14.98 41.6/20.1/10.5/5.8 (BP = 0.996 ratio = 0.996 hyp_len = 111404 ref_len = 111811)\n",
            "2021-08-02 08:56:59 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint10.pt\n",
            "2021-08-02 08:56:59 | INFO | hw5.seq2seq | end of epoch 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e0878781c7b8493ab25445627709bf5f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 11', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:59:01 | INFO | hw5.seq2seq | training loss: 4.2753\n",
            "2021-08-02 08:59:01 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d432d5cd0d8a4d84a9bf9b412058fb50",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 08:59:22 | INFO | hw5.seq2seq | example source: and i was asked to move out because one of my roommates had shared my status with her parents .\n",
            "2021-08-02 08:59:22 | INFO | hw5.seq2seq | example hypothesis: 我被要求移除 , 因為我的房間裡有著她的父母 。\n",
            "2021-08-02 08:59:22 | INFO | hw5.seq2seq | example reference: 我會被要求搬出去 , 是因為其中一名室友和她的父母談了我的狀況 。\n",
            "2021-08-02 08:59:22 | INFO | hw5.seq2seq | validation loss:\t4.0556\n",
            "2021-08-02 08:59:22 | INFO | hw5.seq2seq | BLEU = 15.51 43.1/20.8/11.0/6.1 (BP = 0.989 ratio = 0.989 hyp_len = 110548 ref_len = 111811)\n",
            "2021-08-02 08:59:22 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint11.pt\n",
            "2021-08-02 08:59:22 | INFO | hw5.seq2seq | end of epoch 11\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4f8a2ec038d14d7c96be1286e00884ab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 12', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:01:23 | INFO | hw5.seq2seq | training loss: 4.2349\n",
            "2021-08-02 09:01:23 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0ebad828fc0341c3adb05f06d57ab6ba",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:01:44 | INFO | hw5.seq2seq | example source: and i want to share with you a quick video from our facility that gives you a sense of how this looks at scale .\n",
            "2021-08-02 09:01:44 | INFO | hw5.seq2seq | example hypothesis: 我想跟大家分享一個快速的影片 , 給大家看一下這看法的影片 。\n",
            "2021-08-02 09:01:44 | INFO | hw5.seq2seq | example reference: 我想與你們快速分享一段關於我們設備的影片可以瞭解一下它的規模\n",
            "2021-08-02 09:01:44 | INFO | hw5.seq2seq | validation loss:\t4.0293\n",
            "2021-08-02 09:01:44 | INFO | hw5.seq2seq | BLEU = 15.60 45.3/22.2/11.8/6.5 (BP = 0.934 ratio = 0.937 hyp_len = 104714 ref_len = 111811)\n",
            "2021-08-02 09:01:44 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint12.pt\n",
            "2021-08-02 09:01:44 | INFO | hw5.seq2seq | end of epoch 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61954000e65d45fd9045e4b1b933d624",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 13', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:03:45 | INFO | hw5.seq2seq | training loss: 4.2014\n",
            "2021-08-02 09:03:45 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3e541d4372734057a2cd15328ad0273e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:04:06 | INFO | hw5.seq2seq | example source: it's a distinction perhaps we might make between two nobel laureates , richard feynman and john nash .\n",
            "2021-08-02 09:04:06 | INFO | hw5.seq2seq | example hypothesis: 我們可能在兩位諾貝爾夫婦和約翰納納納納納納許 。\n",
            "2021-08-02 09:04:06 | INFO | hw5.seq2seq | example reference: 我們或許可以從兩位諾貝爾獎得主 , 理查費曼和約翰奈許之間看出這樣的差異 。\n",
            "2021-08-02 09:04:06 | INFO | hw5.seq2seq | validation loss:\t4.0178\n",
            "2021-08-02 09:04:06 | INFO | hw5.seq2seq | BLEU = 15.88 44.6/22.2/11.9/6.6 (BP = 0.953 ratio = 0.954 hyp_len = 106643 ref_len = 111811)\n",
            "2021-08-02 09:04:06 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint13.pt\n",
            "2021-08-02 09:04:06 | INFO | hw5.seq2seq | end of epoch 13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7a3eb3e711b74ff5bf4a2772ee701d27",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 14', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:06:08 | INFO | hw5.seq2seq | training loss: 4.1729\n",
            "2021-08-02 09:06:08 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8cbfc946dab44b07be54ce15b78ff8f7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:06:28 | INFO | hw5.seq2seq | example source: and therein lie the words of wisdom of a young girl whose brief life forever impacted mine .\n",
            "2021-08-02 09:06:28 | INFO | hw5.seq2seq | example hypothesis: 對於年輕女孩來說 , 希望有年輕女孩的夢想 , 會影響我 。\n",
            "2021-08-02 09:06:28 | INFO | hw5.seq2seq | example reference: 她說的話對我有很大啟發 。 這一個年幼女孩短暫的一生永遠影響著我 。\n",
            "2021-08-02 09:06:28 | INFO | hw5.seq2seq | validation loss:\t4.0057\n",
            "2021-08-02 09:06:28 | INFO | hw5.seq2seq | BLEU = 15.36 51.8/26.2/14.1/8.0 (BP = 0.776 ratio = 0.798 hyp_len = 89183 ref_len = 111811)\n",
            "2021-08-02 09:06:28 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint14.pt\n",
            "2021-08-02 09:06:28 | INFO | hw5.seq2seq | end of epoch 14\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bb8dbd0a1a524b9bb6fc442ad469579f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 15', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:08:30 | INFO | hw5.seq2seq | training loss: 4.1519\n",
            "2021-08-02 09:08:30 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "65df5040da69435ba8c20cc8c5e748c8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:08:51 | INFO | hw5.seq2seq | example source: and we've started making curtains , and not only is it beautiful , but people can see status that you care about your children .\n",
            "2021-08-02 09:08:51 | INFO | hw5.seq2seq | example hypothesis: 我們已經開始做曲線 , 不僅是很美麗的 , 還可以看見你在乎你的孩子所關心的人 。\n",
            "2021-08-02 09:08:51 | INFO | hw5.seq2seq | example reference: 我們也開始生産窗簾不止因爲窗簾看起來漂亮 , 人們也可以看到你的地位看得出你很關心小孩\n",
            "2021-08-02 09:08:51 | INFO | hw5.seq2seq | validation loss:\t3.9703\n",
            "2021-08-02 09:08:51 | INFO | hw5.seq2seq | BLEU = 16.50 45.1/22.4/12.0/6.8 (BP = 0.976 ratio = 0.976 hyp_len = 109108 ref_len = 111811)\n",
            "2021-08-02 09:08:51 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint15.pt\n",
            "2021-08-02 09:08:52 | INFO | hw5.seq2seq | end of epoch 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4dd08c8557d047e88e47a9b167ebf612",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 16', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:10:54 | INFO | hw5.seq2seq | training loss: 4.1238\n",
            "2021-08-02 09:10:54 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bec370c73f764f95a7a113ad4da26557",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:11:16 | INFO | hw5.seq2seq | example source: now , i'm not suggesting we want to raise our babies in our stomach , but i am suggesting it's possible we might want to manage gastric secretion in the gut .\n",
            "2021-08-02 09:11:16 | INFO | hw5.seq2seq | example hypothesis: 我不是建議我們要把我們的嬰兒放進我們的嬰兒 , 但是我建議我們要把我們的嬰兒放在腸道上 。\n",
            "2021-08-02 09:11:16 | INFO | hw5.seq2seq | example reference: 我並不是說要把嬰兒養在胃裡而是我們可能可以了解胃在內臟裡的分泌物\n",
            "2021-08-02 09:11:16 | INFO | hw5.seq2seq | validation loss:\t3.9507\n",
            "2021-08-02 09:11:16 | INFO | hw5.seq2seq | BLEU = 16.68 44.2/22.0/11.8/6.7 (BP = 1.000 ratio = 1.006 hyp_len = 112454 ref_len = 111811)\n",
            "2021-08-02 09:11:16 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint16.pt\n",
            "2021-08-02 09:11:16 | INFO | hw5.seq2seq | end of epoch 16\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "52cea86d42b64e91864018672008beb4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 17', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:13:17 | INFO | hw5.seq2seq | training loss: 4.1068\n",
            "2021-08-02 09:13:17 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cf55acb845f540cf87fc0be37b864bee",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:13:39 | INFO | hw5.seq2seq | example source: so we're hoping that's what they'll do .\n",
            "2021-08-02 09:13:39 | INFO | hw5.seq2seq | example hypothesis: 我們希望這就是他們所做的 。\n",
            "2021-08-02 09:13:39 | INFO | hw5.seq2seq | example reference: 所以 , 我們希望它們能夠幫忙 。\n",
            "2021-08-02 09:13:39 | INFO | hw5.seq2seq | validation loss:\t3.9335\n",
            "2021-08-02 09:13:39 | INFO | hw5.seq2seq | BLEU = 16.75 44.8/22.3/12.0/6.7 (BP = 0.993 ratio = 0.993 hyp_len = 111041 ref_len = 111811)\n",
            "2021-08-02 09:13:39 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint17.pt\n",
            "2021-08-02 09:13:39 | INFO | hw5.seq2seq | end of epoch 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5bc2c5b527a64a36abba837721e5f443",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 18', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:15:39 | INFO | hw5.seq2seq | training loss: 4.0873\n",
            "2021-08-02 09:15:39 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "915f8190942f4abaa7b4bcb02a584454",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:16:00 | INFO | hw5.seq2seq | example source: i remember so vividly that day .\n",
            "2021-08-02 09:16:00 | INFO | hw5.seq2seq | example hypothesis: 我還記得很明顯的那一天 。\n",
            "2021-08-02 09:16:00 | INFO | hw5.seq2seq | example reference: 我對那一天還記憶猶新 。\n",
            "2021-08-02 09:16:00 | INFO | hw5.seq2seq | validation loss:\t3.9283\n",
            "2021-08-02 09:16:00 | INFO | hw5.seq2seq | BLEU = 16.70 47.2/23.6/12.7/7.2 (BP = 0.936 ratio = 0.938 hyp_len = 104863 ref_len = 111811)\n",
            "2021-08-02 09:16:00 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint18.pt\n",
            "2021-08-02 09:16:00 | INFO | hw5.seq2seq | end of epoch 18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3cda324cbe904dc8b44d12a9099a5050",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 19', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:18:00 | INFO | hw5.seq2seq | training loss: 4.0703\n",
            "2021-08-02 09:18:00 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20447be793d84d8cac640b5b318941f9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:18:21 | INFO | hw5.seq2seq | example source: the next step is to make compliant toes , and try to add spines and claws and set it for dry adhesives .\n",
            "2021-08-02 09:18:21 | INFO | hw5.seq2seq | example hypothesis: 下一步是要做出合理的玩具 , 試圖加入細菌和爪子 , 試圖加上脊椎和爪子 。\n",
            "2021-08-02 09:18:21 | INFO | hw5.seq2seq | example reference: 下一步是要製造順從聽話的腳趾頭 。 然後加上刺和爪子 , 再加上乾膠 。\n",
            "2021-08-02 09:18:21 | INFO | hw5.seq2seq | validation loss:\t3.9192\n",
            "2021-08-02 09:18:21 | INFO | hw5.seq2seq | BLEU = 17.25 47.6/24.0/13.0/7.4 (BP = 0.949 ratio = 0.950 hyp_len = 106248 ref_len = 111811)\n",
            "2021-08-02 09:18:21 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint19.pt\n",
            "2021-08-02 09:18:21 | INFO | hw5.seq2seq | end of epoch 19\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9657008a39b64ba0af183af35695a0e8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 20', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:20:21 | INFO | hw5.seq2seq | training loss: 4.0546\n",
            "2021-08-02 09:20:21 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3c6f863a92e34b2a9c6bd51b5eaa51a1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:20:42 | INFO | hw5.seq2seq | example source: and got the cord off of the baby's neck , and a healthy screaming , kicking baby arrived , just as the dad ran in from the parking lot , \" hi , you have a son , i'm dr . darria .\n",
            "2021-08-02 09:20:42 | INFO | hw5.seq2seq | example hypothesis: 寶寶的嬰兒跳上了 , 跳舞 , 跳舞 , 跳舞 , 跳舞 , 跳著 , 跳著 , 爸爸在停車場裡 , 說: 「 嗨 , 你有一隻兒子 , 我是爸爸 。 」\n",
            "2021-08-02 09:20:42 | INFO | hw5.seq2seq | example reference: 我把臍帶從嬰兒的脖子繞開 , 接著 , 一個健康哭鬧的嬰兒誕生了 。 同時 , 他的父親剛從停車場趕來 。 「 你好 , 是一個男孩 。 我是達里亞醫生 。\n",
            "2021-08-02 09:20:42 | INFO | hw5.seq2seq | validation loss:\t3.9031\n",
            "2021-08-02 09:20:42 | INFO | hw5.seq2seq | BLEU = 17.16 46.5/23.2/12.5/7.1 (BP = 0.976 ratio = 0.976 hyp_len = 109172 ref_len = 111811)\n",
            "2021-08-02 09:20:42 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint20.pt\n",
            "2021-08-02 09:20:42 | INFO | hw5.seq2seq | end of epoch 20\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6aeee743f1494b29b45a92728701ed04",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 21', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:22:43 | INFO | hw5.seq2seq | training loss: 4.0456\n",
            "2021-08-02 09:22:43 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b02fb5b9ae834ae0a25d7f14c945fb86",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:23:03 | INFO | hw5.seq2seq | example source: no , but the issue of city building in democracy is interesting because it creates chaos , right ?\n",
            "2021-08-02 09:23:03 | INFO | hw5.seq2seq | example hypothesis: 不 , 對嗎 ? 因為它創造了混亂 , 對吧 ?\n",
            "2021-08-02 09:23:03 | INFO | hw5.seq2seq | example reference: 不 , 但民主的城市建設很有趣 , 因為會造成混亂 , 是嗎 ?\n",
            "2021-08-02 09:23:03 | INFO | hw5.seq2seq | validation loss:\t3.8963\n",
            "2021-08-02 09:23:03 | INFO | hw5.seq2seq | BLEU = 17.50 47.3/23.9/13.0/7.4 (BP = 0.963 ratio = 0.963 hyp_len = 107726 ref_len = 111811)\n",
            "2021-08-02 09:23:03 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint21.pt\n",
            "2021-08-02 09:23:03 | INFO | hw5.seq2seq | end of epoch 21\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a724a1d441142b8b9d781d59fd42fa3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 22', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:25:04 | INFO | hw5.seq2seq | training loss: 4.0276\n",
            "2021-08-02 09:25:04 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d190df2951f1490cbf75594585c6be63",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:25:24 | INFO | hw5.seq2seq | example source: so i'm going to show you a demo of this .\n",
            "2021-08-02 09:25:24 | INFO | hw5.seq2seq | example hypothesis: 所以 , 我要給你們看一下這個的示範 。\n",
            "2021-08-02 09:25:24 | INFO | hw5.seq2seq | example reference: 現在請各位看示範\n",
            "2021-08-02 09:25:24 | INFO | hw5.seq2seq | validation loss:\t3.8885\n",
            "2021-08-02 09:25:24 | INFO | hw5.seq2seq | BLEU = 17.38 46.5/23.4/12.7/7.2 (BP = 0.978 ratio = 0.978 hyp_len = 109339 ref_len = 111811)\n",
            "2021-08-02 09:25:24 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint22.pt\n",
            "2021-08-02 09:25:24 | INFO | hw5.seq2seq | end of epoch 22\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bafdbc3df70a469c8b59158e69cbf66e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 23', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:27:25 | INFO | hw5.seq2seq | training loss: 4.0171\n",
            "2021-08-02 09:27:25 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f46a81fe38814412946d54f649cc8c90",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:27:45 | INFO | hw5.seq2seq | example source: you see , in every country where you hear about armed jihadis targeting civilians , there are also unarmed people defying those militants that you don't hear about , and those people need our support to succeed .\n",
            "2021-08-02 09:27:45 | INFO | hw5.seq2seq | example hypothesis: 在每個國家中 , 你聽到的軍隊都沒有聽見 , 有些人不需要支持 , 才能成功 。\n",
            "2021-08-02 09:27:45 | INFO | hw5.seq2seq | example reference: 你看 , 不管在哪個國家你都會聽到武裝聖戰者針對平民百姓 , 也有很多手無寸鐵的人民公然反抗那些激進分子 , 只是你沒聽過 , 那些人需要我們的支持才能成功 。\n",
            "2021-08-02 09:27:45 | INFO | hw5.seq2seq | validation loss:\t3.8931\n",
            "2021-08-02 09:27:45 | INFO | hw5.seq2seq | BLEU = 17.03 50.6/25.9/14.1/8.1 (BP = 0.865 ratio = 0.874 hyp_len = 97690 ref_len = 111811)\n",
            "2021-08-02 09:27:45 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint23.pt\n",
            "2021-08-02 09:27:45 | INFO | hw5.seq2seq | end of epoch 23\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f7c7f6dca238412898b6b8d8ed4d9b9f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 24', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:29:46 | INFO | hw5.seq2seq | training loss: 4.0046\n",
            "2021-08-02 09:29:46 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ca1145b5312846519a7401f51052274b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:30:06 | INFO | hw5.seq2seq | example source: from gezi to tahrir to elsewhere , i've seen people put their lives and livelihoods on the line .\n",
            "2021-08-02 09:30:06 | INFO | hw5.seq2seq | example hypothesis: 從gezi到其他地方 , 我見過人們把他們的生活和生存放在線上 。\n",
            "2021-08-02 09:30:06 | INFO | hw5.seq2seq | example reference: 從格濟公園 , 到解放廣場 , 再到其它地方 , 我見過許多人用他們的生命和生計做賭注 。\n",
            "2021-08-02 09:30:06 | INFO | hw5.seq2seq | validation loss:\t3.8744\n",
            "2021-08-02 09:30:06 | INFO | hw5.seq2seq | BLEU = 17.50 47.3/23.8/12.9/7.4 (BP = 0.966 ratio = 0.967 hyp_len = 108098 ref_len = 111811)\n",
            "2021-08-02 09:30:06 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint24.pt\n",
            "2021-08-02 09:30:06 | INFO | hw5.seq2seq | end of epoch 24\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9c1fd904008c480c87eb03a586c29bc2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 25', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:32:07 | INFO | hw5.seq2seq | training loss: 3.9973\n",
            "2021-08-02 09:32:07 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a0211a22604248319676f38dc22a78a8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:32:28 | INFO | hw5.seq2seq | example source: and every project before that had been completely personal and it was a revelation when people just started commenting , started giving feedback on your code .\n",
            "2021-08-02 09:32:28 | INFO | hw5.seq2seq | example hypothesis: 而每個專案都已經完全個人化了 , 開始給予回饋信號時 , 開始給予回饋 。\n",
            "2021-08-02 09:32:28 | INFO | hw5.seq2seq | example reference: 在那之前每一件計劃都是我個人的東西 , 所以當大家開始評論 , 開始對你的程式給意見時 , 真的是一種啟示 。\n",
            "2021-08-02 09:32:28 | INFO | hw5.seq2seq | validation loss:\t3.8623\n",
            "2021-08-02 09:32:28 | INFO | hw5.seq2seq | BLEU = 17.63 46.2/23.3/12.6/7.1 (BP = 0.999 ratio = 0.999 hyp_len = 111668 ref_len = 111811)\n",
            "2021-08-02 09:32:28 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint25.pt\n",
            "2021-08-02 09:32:28 | INFO | hw5.seq2seq | end of epoch 25\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d6f284b48d1a485fb29f8b0081076c7b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 26', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:34:29 | INFO | hw5.seq2seq | training loss: 3.9879\n",
            "2021-08-02 09:34:29 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20797fca24d54f9e8fda279baa80ddf6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:34:48 | INFO | hw5.seq2seq | example source: thank you so much .\n",
            "2021-08-02 09:34:48 | INFO | hw5.seq2seq | example hypothesis: 謝謝\n",
            "2021-08-02 09:34:48 | INFO | hw5.seq2seq | example reference: 感謝各位.\n",
            "2021-08-02 09:34:48 | INFO | hw5.seq2seq | validation loss:\t3.8676\n",
            "2021-08-02 09:34:48 | INFO | hw5.seq2seq | BLEU = 17.37 51.2/26.3/14.4/8.3 (BP = 0.869 ratio = 0.877 hyp_len = 98030 ref_len = 111811)\n",
            "2021-08-02 09:34:48 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint26.pt\n",
            "2021-08-02 09:34:48 | INFO | hw5.seq2seq | end of epoch 26\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "18272d2a0d7f4471aff9df12797a5fae",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 27', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:36:49 | INFO | hw5.seq2seq | training loss: 3.9813\n",
            "2021-08-02 09:36:49 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a38bd76094564130b2d09ff8cc8ec220",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:37:09 | INFO | hw5.seq2seq | example source: at the age of six months , virtually every one of us is able to differentiate between animate and inanimate objects .\n",
            "2021-08-02 09:37:09 | INFO | hw5.seq2seq | example hypothesis: 在六個月內 , 我們每一個人都可以分辨出與自動之間的分類 。\n",
            "2021-08-02 09:37:09 | INFO | hw5.seq2seq | example reference: 在六個月大時 , 幾乎每個人都能辨別東西是否有生命 。\n",
            "2021-08-02 09:37:09 | INFO | hw5.seq2seq | validation loss:\t3.8588\n",
            "2021-08-02 09:37:09 | INFO | hw5.seq2seq | BLEU = 17.90 47.1/24.1/13.1/7.5 (BP = 0.979 ratio = 0.980 hyp_len = 109526 ref_len = 111811)\n",
            "2021-08-02 09:37:10 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint27.pt\n",
            "2021-08-02 09:37:10 | INFO | hw5.seq2seq | end of epoch 27\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "762695fe4746402d8dcbbbb0bab9c42a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 28', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:39:11 | INFO | hw5.seq2seq | training loss: 3.9694\n",
            "2021-08-02 09:39:11 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "982b55b923244a728248efecaec4e704",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:39:32 | INFO | hw5.seq2seq | example source: the problem is there is no such thing as a viable democracy made up of experts , zealots , politicians and spectators .\n",
            "2021-08-02 09:39:32 | INFO | hw5.seq2seq | example hypothesis: 問題在於 , 政治人物和觀點都無法實現 。\n",
            "2021-08-02 09:39:32 | INFO | hw5.seq2seq | example reference: 事實上 , 根本沒有一種民主制度是由專家 , 狂熱分子 , 政治家和旁觀者組成的\n",
            "2021-08-02 09:39:32 | INFO | hw5.seq2seq | validation loss:\t3.8506\n",
            "2021-08-02 09:39:32 | INFO | hw5.seq2seq | BLEU = 17.82 48.3/24.5/13.3/7.7 (BP = 0.956 ratio = 0.957 hyp_len = 106987 ref_len = 111811)\n",
            "2021-08-02 09:39:32 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint28.pt\n",
            "2021-08-02 09:39:32 | INFO | hw5.seq2seq | end of epoch 28\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d2257c0520d1499898c304ac5dac213a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 29', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:41:33 | INFO | hw5.seq2seq | training loss: 3.9626\n",
            "2021-08-02 09:41:33 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "20ffd436d8694784a86a1e48f4e08eb2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:41:54 | INFO | hw5.seq2seq | example source: they're learning our culture , our language , our heritage and realizing we're just as different and just the same as each other .\n",
            "2021-08-02 09:41:54 | INFO | hw5.seq2seq | example hypothesis: 他們在學習我們的文化、我們的文化、我們的語言、我們的文化、我們的語言、我們的文化、我們的文化、我們的文化、我們的文化、我們的文化、我們的文化、我們的文化、我們的文化、我們的文化、我們的文化 , 都不一樣 ,\n",
            "2021-08-02 09:41:54 | INFO | hw5.seq2seq | example reference: 他們正在習知我們的文化、語言、傳統 , 並意識到彼此不同及相同之處 。\n",
            "2021-08-02 09:41:54 | INFO | hw5.seq2seq | validation loss:\t3.8419\n",
            "2021-08-02 09:41:54 | INFO | hw5.seq2seq | BLEU = 18.01 47.1/23.9/13.0/7.5 (BP = 0.990 ratio = 0.990 hyp_len = 110701 ref_len = 111811)\n",
            "2021-08-02 09:41:54 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint29.pt\n",
            "2021-08-02 09:41:55 | INFO | hw5.seq2seq | end of epoch 29\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "db4e2028ea3d4a1e9af1bbb3c5d87168",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='train epoch 30', max=791.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:43:56 | INFO | hw5.seq2seq | training loss: 3.9542\n",
            "2021-08-02 09:43:56 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5894d1718d74d66a39e4e0c4750ceb8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:44:17 | INFO | hw5.seq2seq | example source: but did you know that only two percent of that funding went to rebuild haitian public institutions , including its health sector ?\n",
            "2021-08-02 09:44:17 | INFO | hw5.seq2seq | example hypothesis: 但你知道只有2%的資金來重建美國公共機構 , 包含健康部門嗎 ?\n",
            "2021-08-02 09:44:17 | INFO | hw5.seq2seq | example reference: 但你知不知道國際社會捐款中只有2%用去重建海地的公共體制 , 包括衞生部門 ?\n",
            "2021-08-02 09:44:17 | INFO | hw5.seq2seq | validation loss:\t3.8446\n",
            "2021-08-02 09:44:17 | INFO | hw5.seq2seq | BLEU = 18.02 48.1/24.6/13.5/7.8 (BP = 0.960 ratio = 0.960 hyp_len = 107372 ref_len = 111811)\n",
            "2021-08-02 09:44:17 | INFO | hw5.seq2seq | saved epoch checkpoint: /content/checkpoints/rnn/checkpoint30.pt\n",
            "2021-08-02 09:44:17 | INFO | hw5.seq2seq | end of epoch 30\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LnqAyTDDpP_P"
      },
      "source": [
        "# Submission 繳交檔案"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3O7NSDdepP_P",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f28e1ad-8b4b-4977-8549-213b3a1dc06d"
      },
      "source": [
        "# 把幾個 checkpoint 平均起來可以達到 ensemble 的效果\n",
        "checkdir=config.savedir\n",
        "!python ./fairseq/scripts/average_checkpoints.py \\\n",
        "--inputs {checkdir} \\\n",
        "--num-epoch-checkpoints 5 \\\n",
        "--output {checkdir}/avg_last_5_checkpoint.pt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Namespace(checkpoint_upper_bound=None, inputs=['./checkpoints/rnn'], num_epoch_checkpoints=5, num_update_checkpoints=None, output='./checkpoints/rnn/avg_last_5_checkpoint.pt')\n",
            "averaging checkpoints:  ['./checkpoints/rnn/checkpoint30.pt', './checkpoints/rnn/checkpoint29.pt', './checkpoints/rnn/checkpoint28.pt', './checkpoints/rnn/checkpoint27.pt', './checkpoints/rnn/checkpoint26.pt']\n",
            "Finished writing averaged checkpoint to ./checkpoints/rnn/avg_last_5_checkpoint.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "syWKeRNPpP_P"
      },
      "source": [
        "## 確認生成繳交檔案的模型參數"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eP8hB67lpP_P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163,
          "referenced_widgets": [
            "f1b999b1c89a4bd29c66e174a6d1dd63",
            "b97ad6be6858426ea619cbe0ffc57de3",
            "791cfcee5ff64a8d8c91643e82d803ac",
            "0a48a75124584455aeb1fc8fd64ea6fd",
            "6e3c6e0e6c8e4cc190c31ca75ba848e6",
            "8883c7ab13c6432890a931c54550aded",
            "baff250cabbd4c418cc72bd544d5c1d4",
            "22481d05b1834c29af5864445771b99c"
          ]
        },
        "outputId": "b9676443-a477-4de5-fe75-a21337c3d21a"
      },
      "source": [
        "# checkpoint_last.pt : 最後一次檢驗的檔案\n",
        "# checkpoint_best.pt : 檢驗 BLEU 最高的檔案\n",
        "# avg_last_5_checkpoint.pt:　最5後個檔案平均\n",
        "try_load_checkpoint(model, name=\"avg_last_5_checkpoint.pt\")\n",
        "validate(model, task, criterion, log_to_wandb=False)\n",
        "None"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 09:44:55 | INFO | hw5.seq2seq | loaded checkpoint checkpoints/rnn/avg_last_5_checkpoint.pt: step=unknown loss=3.844637870788574 bleu=18.021526251025\n",
            "2021-08-02 09:44:55 | INFO | hw5.seq2seq | begin validation\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f1b999b1c89a4bd29c66e174a6d1dd63",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='validation', max=22.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r2021-08-02 09:45:15 | INFO | hw5.seq2seq | example source: thank you .\n",
            "2021-08-02 09:45:15 | INFO | hw5.seq2seq | example hypothesis: 謝謝\n",
            "2021-08-02 09:45:15 | INFO | hw5.seq2seq | example reference: 謝謝 。\n",
            "2021-08-02 09:45:16 | INFO | hw5.seq2seq | validation loss:\t3.8116\n",
            "2021-08-02 09:45:16 | INFO | hw5.seq2seq | BLEU = 18.20 49.0/25.2/13.8/8.0 (BP = 0.946 ratio = 0.947 hyp_len = 105934 ref_len = 111811)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5P8g6cX8pP_Q"
      },
      "source": [
        "## 進行預測"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGP6QOihpP_Q"
      },
      "source": [
        "def generate_prediction(model, task, split=\"test\", outfile=\"./prediction.txt\"):    \n",
        "    task.load_dataset(split=split, epoch=1)\n",
        "    itr = load_data_iterator(task, split, 1, config.max_tokens, config.num_workers).next_epoch_itr(shuffle=False)\n",
        "    \n",
        "    idxs = []\n",
        "    hyps = []\n",
        "\n",
        "    model.eval()\n",
        "    progress = tqdm.tqdm(itr, desc=f\"prediction\")\n",
        "    with torch.no_grad():\n",
        "        for i, sample in enumerate(progress):\n",
        "            # validation loss\n",
        "            sample = utils.move_to_cuda(sample, device=device)\n",
        "\n",
        "            # 進行推論\n",
        "            s, h, r = inference_step(sample, model)\n",
        "            \n",
        "            hyps.extend(h)\n",
        "            idxs.extend(list(sample['id']))\n",
        "            \n",
        "    # 根據 preprocess 時的順序排列\n",
        "    hyps = [x for _,x in sorted(zip(idxs,hyps))]\n",
        "    \n",
        "    with open(outfile, \"w\") as f:\n",
        "        for h in hyps:\n",
        "            f.write(h+\"\\n\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TzagK0gGpP_Q",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "55110f581fae40fc86a88e6a7f119b37",
            "97751f3c5cd840b583ee02157a63bc30",
            "6ac17b1b052d4d80bde3e3c285663455",
            "2e9a65851bd34dd4b248db6608cf66d4",
            "e87f3d19fa4a431f9511a9b0ae6f8d72",
            "0371e2532a92493c9488b22773264212",
            "937620760b354db1a6154f06badfe542",
            "6ee342183b5545db8b51db81043a17d8"
          ]
        },
        "outputId": "412fc0f0-a6c0-4e37-ab9e-d4b3cd2b0f30"
      },
      "source": [
        "generate_prediction(model, task)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-08-02 09:45:33 | INFO | fairseq.data.data_utils | loaded 4,000 examples from: ./DATA/data-bin/ted2020/test.en-zh.en\n",
            "2021-08-02 09:45:33 | INFO | fairseq.data.data_utils | loaded 4,000 examples from: ./DATA/data-bin/ted2020/test.en-zh.zh\n",
            "2021-08-02 09:45:33 | INFO | fairseq.tasks.translation | ./DATA/data-bin/ted2020 test en-zh 4000 examples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "55110f581fae40fc86a88e6a7f119b37",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='prediction', max=17.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QH5XujTipP_Q"
      },
      "source": [
        "raise"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cml77b_GpP_R"
      },
      "source": [
        "# Back-translation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gs0rD5vMpP_R"
      },
      "source": [
        "## 訓練一個反向的翻譯模型"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHm7-qaWpP_R"
      },
      "source": [
        "1. 將實驗的參數設定表中(config)的source_lang與target_lang互相交換\n",
        "2. 將實驗的參數設定表中(config)的savedir更改(ex. \"./checkpoints/rnn-back\")\n",
        "3. 訓練一個反向模型"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SiNt9FUzpP_R"
      },
      "source": [
        "## 利用反向模型生成額外資料"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AGlJU7zMpP_R"
      },
      "source": [
        "### 下載 monolingual data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQnIEL7HpP_S"
      },
      "source": [
        "mono_dataset_name = 'mono'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8lS8ecZJpP_S"
      },
      "source": [
        "mono_prefix = Path(data_dir).absolute() / mono_dataset_name\n",
        "mono_prefix.mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "urls = (\n",
        "    '\"https://onedrive.live.com/download?cid=3E549F3B24B238B4&resid=3E549F3B24B238B4%214986&authkey=AANUKbGfZx0kM80\"',\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://www.csie.ntu.edu.tw/~r09922057/ML2021-hw5/ted_zh_corpus.deduped.gz\",\n",
        "# # If the above links die, use the following instead. \n",
        "#     \"https://mega.nz/#!vMNnDShR!4eHDxzlpzIpdpeQTD-htatU_C7QwcBTwGDaSeBqH534\",\n",
        ")\n",
        "file_names = (\n",
        "    'ted_zh_corpus.deduped.gz',\n",
        ")\n",
        "\n",
        "for u, f in zip(urls, file_names):\n",
        "    path = mono_prefix/f\n",
        "    if not path.exists():\n",
        "        if 'mega' in u:\n",
        "            !megadl {u} --path {path}\n",
        "        else:\n",
        "            !wget {u} -O {path}\n",
        "    else:\n",
        "        print(f'{f} is exist, skip downloading')\n",
        "    if path.suffix == \".tgz\":\n",
        "        !tar -xvf {path} -C {prefix}\n",
        "    elif path.suffix == \".zip\":\n",
        "        !unzip -o {path} -d {prefix}\n",
        "    elif path.suffix == \".gz\":\n",
        "        !gzip -fkd {path}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVkHLwpYpP_S"
      },
      "source": [
        "### TODO: 清理資料集\n",
        "\n",
        "1. 將太長、太短的句子移除\n",
        "2. 統一標點符號\n",
        "\n",
        "hint: 可以使用clean_s()來協助"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7jxsGbUpP_S"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VIDJl34epP_T"
      },
      "source": [
        "### TODO: Subword Units\n",
        "\n",
        "用反向模型的 spm model 將資料切成 subword units\n",
        "\n",
        "hint: spm model 的路徑為 DATA/raw-data/\\[dataset\\]/spm\\[vocab_num\\].model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nYUvoNqApP_T"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zPsxOqoJpP_T"
      },
      "source": [
        "### Binarize\n",
        "\n",
        "使用fairseq將資料轉為binary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8EZPJXYMpP_T"
      },
      "source": [
        "binpath = Path('./DATA/data-bin', mono_dataset_name)\n",
        "src_dict_file = './DATA/data-bin/ted2020/dict.en.txt'\n",
        "tgt_dict_file = src_dict_file\n",
        "monopref = str(mono_prefix/\"mono.tok\") # whatever filepath you get after applying subword tokenization\n",
        "if binpath.exists():\n",
        "    print(binpath, \"exists, will not overwrite!\")\n",
        "else:\n",
        "    !python -m fairseq_cli.preprocess\\\n",
        "        --source-lang 'zh'\\\n",
        "        --target-lang 'en'\\\n",
        "        --trainpref {monopref}\\\n",
        "        --destdir {binpath}\\\n",
        "        --srcdict {src_dict_file}\\\n",
        "        --tgtdict {tgt_dict_file}\\\n",
        "        --workers 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bFaBn9gypP_T"
      },
      "source": [
        "### TODO: 生成反向翻譯資料\n",
        "\n",
        "將 binarized data 加入原本的資料夾中並用一個 split_name 取名\n",
        "\n",
        "ex. ./DATA/data-bin/ted2020/\\[split_name\\].zh-en.\\[\"en\", \"zh\"\\].\\[\"bin\", \"idx\"\\]\n",
        "\n",
        "便可以使用 generate_prediction(model, task, split=\"split_name\")來產生翻譯資料"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePmzhiQlpP_U"
      },
      "source": [
        "# 將 binarized data 加入原本的資料夾中並用一個 split_name 取名\n",
        "# ex. ./DATA/data-bin/ted2020/\\[split_name\\].zh-en.\\[\"en\", \"zh\"\\].\\[\"bin\", \"idx\"\\]\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.zh.bin ./DATA/data-bin/ted2020/mono.zh-en.zh.bin\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.zh.idx ./DATA/data-bin/ted2020/mono.zh-en.zh.idx\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.en.bin ./DATA/data-bin/ted2020/mono.zh-en.en.bin\n",
        "!cp ./DATA/data-bin/mono/train.zh-en.en.idx ./DATA/data-bin/ted2020/mono.zh-en.en.idx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NoQeVtT6pP_U"
      },
      "source": [
        "# hint: 用反向模型在 split='mono' 上進行預測，生成 prediction_file\n",
        "# generate_prediction( ... ,split=... ,outfile=... )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k7E4Tq9NpP_U"
      },
      "source": [
        "### TODO: 產生新的dataset\n",
        "\n",
        "1. 將翻譯出來的資料與原先的訓練資料結合\n",
        "2. 使用之前的spm model切出成Subword Units\n",
        "3. 重新使用fairseq將資料轉為binary"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3CP9u-KCpP_U"
      },
      "source": [
        "# 合併剛剛生成的 prediction_file (.en) 以及中文 mono.zh (.zh)\n",
        "# \n",
        "# hint: 在此用剛剛的 spm model 對 prediction_file 進行切斷詞\n",
        "# spm_model.encode(line, out_type=str)\n",
        "# output: ./DATA/rawdata/mono/mono.tok.en & mono.tok.zh\n",
        "#\n",
        "# hint: 在此用 fairseq 把這些檔案再 binarize\n",
        "# binpath = Path('./DATA/data-bin/synthetic')\n",
        "# src_dict_file = './DATA/data-bin/ted2020/dict.en.txt'\n",
        "# tgt_dict_file = src_dict_file\n",
        "# monopref = ./DATA/rawdata/mono/mono.tok # or whatever path after applying subword tokenization, w/o the suffix (.zh/.en)\n",
        "# if binpath.exists():\n",
        "#     print(binpath, \"exists, will not overwrite!\")\n",
        "# else:\n",
        "#     !python -m fairseq_cli.preprocess\\\n",
        "#         --source-lang 'zh'\\\n",
        "#         --target-lang 'en'\\\n",
        "#         --trainpref {monopref}\\\n",
        "#         --destdir {binpath}\\\n",
        "#         --srcdict {src_dict_file}\\\n",
        "#         --tgtdict {tgt_dict_file}\\\n",
        "#         --workers 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F-GTT4sypP_V"
      },
      "source": [
        "# 這裡用剛剛準備的檔案合併原先 ted2020 來生成最終 back-translation 的資料\n",
        "!cp -r ./DATA/data-bin/ted2020/ ./DATA/data-bin/ted2020_with_mono/\n",
        "\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.zh.bin ./DATA/data-bin/ted2020_with_mono/train1.en-zh.zh.bin\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.zh.idx ./DATA/data-bin/ted2020_with_mono/train1.en-zh.zh.idx\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.en.bin ./DATA/data-bin/ted2020_with_mono/train1.en-zh.en.bin\n",
        "!cp ./DATA/data-bin/synthetic/train.zh-en.en.idx ./DATA/data-bin/ted2020_with_mono/train1.en-zh.en.idx"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BIJ_lVn4pP_V"
      },
      "source": [
        "### TODO: 重新訓練\n",
        "\n",
        "當已經產生新的資料集\n",
        "\n",
        "1. 將實驗的參數設定表(config)中的datadir改為新的資料集(\"./DATA/data-bin/ted2020_with_mono\")\n",
        "2. 將實驗的參數設定表(config)中的source_lang與target_lang設定還原(\"en\", \"zh\")\n",
        "3. 將實驗的參數設定表(config)中的savedir更改(ex. \"./checkpoints/rnn-bt\")\n",
        "4. 重新訓練"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aT-NU6mypP_V"
      },
      "source": [
        "# References"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vAvFeEQ2pP_W"
      },
      "source": [
        "1. <a name=ott2019fairseq></a>Ott, M., Edunov, S., Baevski, A., Fan, A., Gross, S., Ng, N., ... & Auli, M. (2019, June). fairseq: A Fast, Extensible Toolkit for Sequence Modeling. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics (Demonstrations) (pp. 48-53).\n",
        "2. <a name=vaswani2017></a>Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., ... & Polosukhin, I. (2017, December). Attention is all you need. In Proceedings of the 31st International Conference on Neural Information Processing Systems (pp. 6000-6010).\n",
        "3. <a name=reimers-2020-multilingual-sentence-bert></a>Reimers, N., & Gurevych, I. (2020, November). Making Monolingual Sentence Embeddings Multilingual Using Knowledge Distillation. In Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP) (pp. 4512-4525).\n",
        "4. <a name=tiedemann2012parallel></a>Tiedemann, J. (2012, May). Parallel Data, Tools and Interfaces in OPUS. In Lrec (Vol. 2012, pp. 2214-2218).\n",
        "5. <a name=kudo-richardson-2018-sentencepiece></a>Kudo, T., & Richardson, J. (2018, November). SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing: System Demonstrations (pp. 66-71).\n",
        "6. <a name=sennrich-etal-2016-improving></a>Sennrich, R., Haddow, B., & Birch, A. (2016, August). Improving Neural Machine Translation Models with Monolingual Data. In Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) (pp. 86-96).\n",
        "7. <a name=edunov-etal-2018-understanding></a>Edunov, S., Ott, M., Auli, M., & Grangier, D. (2018). Understanding Back-Translation at Scale. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing (pp. 489-500).\n",
        "8. https://github.com/ajinkyakulkarni14/TED-Multilingual-Parallel-Corpus\n",
        "9. https://ithelp.ithome.com.tw/articles/10233122\n",
        "10. https://nlp.seas.harvard.edu/2018/04/03/attention.html"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dh7TgKRupP_W"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}